rocketjob 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c742a1c093d122fbb2d6d9b194607d1ebd6743f3
-  data.tar.gz: 2998a334bf98a4624ac3c42b69c70c1a4a4dcaf8
+  metadata.gz: 201f922d37f2533e56109c053e3c9f4bdef30108
+  data.tar.gz: 00598e8fb3039d7ce629f629bf270d3cc06f2e87
 SHA512:
-  metadata.gz: b032a72a38db2aeb1f121cd65e5835fce438d77c1f6248560c40086fe4fb47f69815c83b8ac491188819ad0b4fea2a93e661015d7e38e0ccb1474e9b3e6c9505
-  data.tar.gz: f152e0cbd1af97f02674a5b47e8f9854f7cfac1c5fc509489b5ab30cd3c464ba11fc6ba41809687720a2caa18fd8dc2b7eaebc5a33fac2c6df7f411777089960
+  metadata.gz: 447f63cf687c4e7ac10189aff59730874ea25f3e57b7d6ca301fd4a34178e91d68d4428f5be095c2a6a4a5426d603262c3d82c78e9f3988a9ed35d76bf75bddf
+  data.tar.gz: e6a54a1b72cb2c7192894140eea4e49e42dc29049a85a77338de773e539c23a5e1ed09a5691a0a5c238a881e99398cb07b99eddf5abdda8e044971cf433766f3

data/lib/rocket_job/dirmon_entry.rb

@@ -213,30 +213,38 @@ module RocketJob
     @@default_archive_directory = '_archive'.freeze
 
     # Returns [Pathname] the archive_directory if set, otherwise the default_archive_directory
-    def archive_pathname
-      Pathname.new(archive_directory || self.class.default_archive_directory)
+    # Creates the archive directory if one is set
+    def archive_pathname(file_pathname)
+      if archive_directory
+        path = Pathname.new(archive_directory)
+        path.mkpath unless path.exist?
+        path.realpath
+      else
+        file_pathname.dirname.join(self.class.default_archive_directory).realdirpath
+      end
     end
 
     # Passes each filename [Pathname] found that matches the pattern into the supplied block
     def each(&block)
       logger.tagged("DirmonEntry:#{id}") do
-        Pathname.glob(pattern).each do |pathname|
+        # Case insensitive filename matching
+        Pathname.glob(pattern, File::FNM_CASEFOLD).each do |pathname|
           next if pathname.directory?
           pathname  = pathname.realpath
           file_name = pathname.to_s
 
           # Skip archive directories
-          next if file_name.start_with?(archive_pathname.realpath.to_s)
+          next if file_name.include?(self.class.default_archive_directory)
 
           # Security check?
-          if (@@whitelist_paths.size > 0) && @@whitelist_paths.none? { |whitepath| file_name.start_with?(whitepath) }
-            logger.warn "Ignoring file: #{file_name} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
+          if (whitelist_paths.size > 0) && whitelist_paths.none? { |whitepath| file_name.start_with?(whitepath) }
+            logger.error "Skipping file: #{file_name} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
             next
           end
 
           # File must be writable so it can be removed after processing
           unless pathname.writable?
-            logger.warn "Ignoring file: #{file_name} since it is not writable by the current user. Must be able to delete/move the file after queueing the job"
+            logger.error "Skipping file: #{file_name} since it is not writable by the current user. Must be able to delete/move the file after queueing the job"
             next
           end
           block.call(pathname)
@@ -283,6 +291,11 @@ module RocketJob
 
     protected
 
+    # Instance method to return whitelist paths
+    def whitelist_paths
+      @@whitelist_paths
+    end
+
     # Upload the file to the job
     def upload_file(job, pathname)
       if job.respond_to?(:file_store_upload)
@@ -316,7 +329,7 @@ module RocketJob
     # Note:
     # - Works across partitions when the file and the archive are on different partitions
     def archive_file(job, pathname)
-      target_path = archive_pathname
+      target_path = archive_pathname(pathname)
       target_path.mkpath
       target_file_name = target_path.join("#{job.id}_#{pathname.basename}")
       # In case the file is being moved across partitions

data/lib/rocket_job/version.rb

@@ -1,4 +1,4 @@
 # encoding: UTF-8
 module RocketJob #:nodoc
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 end

data/test/dirmon_entry_test.rb

@@ -63,7 +63,7 @@ class DirmonEntryTest < Minitest::Test
 
     describe '#fail_with_exception!' do
       before do
-        @dirmon_entry = RocketJob::DirmonEntry.new(job_class_name: 'Jobs::TestJob', pattern: '/abc/**', arguments: [1])
+        @dirmon_entry = RocketJob::DirmonEntry.new(job_class_name: 'Jobs::TestJob', pattern: 'test/files/**', arguments: [1])
         @dirmon_entry.enable!
       end
       after do
@@ -111,7 +111,7 @@ class DirmonEntryTest < Minitest::Test
 
       describe 'job_class_name' do
         it 'ensure presence' do
-          assert entry = RocketJob::DirmonEntry.new(pattern: '/abc/**')
+          assert entry = RocketJob::DirmonEntry.new(pattern: 'test/files/**')
           assert_equal false, entry.valid?
           assert_equal ["can't be blank", 'job_class_name must be defined and must be derived from RocketJob::Job'], entry.errors[:job_class_name], entry.errors.inspect
         end
@@ -121,7 +121,7 @@ class DirmonEntryTest < Minitest::Test
         it 'allow no arguments' do
           assert entry = RocketJob::DirmonEntry.new(
               job_class_name: 'Jobs::TestJob',
-              pattern:        '/abc/**',
+              pattern:        'test/files/**',
               perform_method: :result
             )
           assert_equal true, entry.valid?, entry.errors.inspect
@@ -131,7 +131,7 @@ class DirmonEntryTest < Minitest::Test
         it 'ensure correct number of arguments' do
           assert entry = RocketJob::DirmonEntry.new(
               job_class_name: 'Jobs::TestJob',
-              pattern:        '/abc/**'
+              pattern:        'test/files/**'
             )
           assert_equal false, entry.valid?
           assert_equal ['There must be 1 argument(s)'], entry.errors[:arguments], entry.errors.inspect
@@ -140,7 +140,7 @@ class DirmonEntryTest < Minitest::Test
         it 'return false if the job name is bad' do
           assert entry = RocketJob::DirmonEntry.new(
               job_class_name: 'Jobs::Tests::Names::Things',
-              pattern:        '/abc/**'
+              pattern:        'test/files/**'
             )
           assert_equal false, entry.valid?
           assert_equal [], entry.errors[:arguments], entry.errors.inspect
@@ -150,7 +150,7 @@ class DirmonEntryTest < Minitest::Test
       it 'arguments with perform_method' do
         assert entry = RocketJob::DirmonEntry.new(
             job_class_name: 'Jobs::TestJob',
-            pattern:        '/abc/**',
+            pattern:        'test/files/**',
             perform_method: :sum
           )
         assert_equal false, entry.valid?
@@ -160,7 +160,7 @@ class DirmonEntryTest < Minitest::Test
       it 'valid' do
         assert entry = RocketJob::DirmonEntry.new(
             job_class_name: 'Jobs::TestJob',
-            pattern:        '/abc/**',
+            pattern:        'test/files/**',
             arguments:      [1]
           )
         assert entry.valid?, entry.errors.inspect
@@ -169,7 +169,7 @@ class DirmonEntryTest < Minitest::Test
       it 'valid with perform_method' do
         assert entry = RocketJob::DirmonEntry.new(
             job_class_name: 'Jobs::TestJob',
-            pattern:        '/abc/**',
+            pattern:        'test/files/**',
             perform_method: :sum,
             arguments:      [1, 2]
           )
@@ -180,20 +180,23 @@ class DirmonEntryTest < Minitest::Test
     describe 'with valid entry' do
       before do
         @archive_directory = '/tmp/archive_directory'
-        @entry             = RocketJob::DirmonEntry.new(
-          pattern:           'abc/*',
+        @archive_path      = Pathname.new(@archive_directory)
+        @archive_path.mkpath
+        @archive_path = @archive_path.realdirpath
+        @entry        = RocketJob::DirmonEntry.new(
+          pattern:           'test/files/**/*',
           job_class_name:    'Jobs::TestJob',
           arguments:         [{input: 'yes'}],
           properties:        {priority: 23, perform_method: :event},
           archive_directory: @archive_directory
         )
-        @job               = Jobs::TestJob.new
-        @file              = Tempfile.new('archive')
-        @file_name         = @file.path
-        @pathname          = Pathname.new(@file_name)
+        @job          = Jobs::TestJob.new
+        @file         = Tempfile.new('archive')
+        @file_name    = @file.path
+        @pathname     = Pathname.new(@file_name)
         File.open(@file_name, 'w') { |file| file.write('Hello World') }
         assert File.exists?(@file_name)
-        @archive_file_name = File.join(@archive_directory, "#{@job.id}_#{File.basename(@file_name)}")
+        @archive_real_name = @archive_path.join("#{@job.id}_#{File.basename(@file_name)}").to_s
       end
 
       after do
@@ -202,26 +205,26 @@ class DirmonEntryTest < Minitest::Test
 
       describe '#archive_pathname' do
         it 'with archive directory' do
-          assert_equal @archive_directory.to_s, @entry.archive_pathname.to_s
+          assert_equal File.dirname(@archive_real_name), @entry.archive_pathname(@pathname).to_s
         end
 
         it 'without archive directory' do
           @entry.archive_directory = nil
-          assert_equal '_archive', @entry.archive_pathname.to_s
+          assert @entry.archive_pathname(@pathname).to_s.end_with?('_archive')
         end
       end
 
       describe '#archive_file' do
         it 'archive file' do
-          assert_equal @archive_file_name, @entry.send(:archive_file, @job, Pathname.new(@file_name))
-          assert File.exists?(@archive_file_name), @archive_file_name
+          assert_equal @archive_real_name, @entry.send(:archive_file, @job, Pathname.new(@file_name))
+          assert File.exists?(@archive_real_name)
         end
       end
 
       describe '#upload_default' do
         it 'upload' do
           @entry.send(:upload_default, @job, @pathname)
-          assert_equal File.absolute_path(@archive_file_name), @job.arguments.first[:full_file_name], @job.arguments
+          assert_equal @archive_real_name, @job.arguments.first[:full_file_name], @job.arguments
         end
       end
 
@@ -248,10 +251,67 @@ class DirmonEntryTest < Minitest::Test
           @entry.arguments      = [{}]
           @entry.perform_method = :event
           job                   = @entry.later(@pathname)
-          assert_equal File.join(@archive_directory, "#{job.id}_#{File.basename(@file_name)}"), job.arguments.first[:full_file_name]
+          assert_equal Pathname.new(@archive_directory).join("#{job.id}_#{File.basename(@file_name)}").realdirpath.to_s, job.arguments.first[:full_file_name]
           assert job.queued?
         end
       end
+
+      describe '#each' do
+        it 'without archive path' do
+          @entry.archive_directory = nil
+          files                    = []
+          @entry.each do |file_name|
+            files << file_name
+          end
+          assert_equal nil, @entry.archive_directory
+          assert_equal 1, files.count
+          assert_equal Pathname.new('test/files/text.txt').realpath, files.first
+        end
+
+        it 'with archive path' do
+          files = []
+          @entry.each do |file_name|
+            files << file_name
+          end
+          assert_equal 1, files.count
+          assert_equal Pathname.new('test/files/text.txt').realpath, files.first
+        end
+
+        it 'with case-insensitive pattern' do
+          @entry.pattern = 'test/files/**/*.TxT'
+          files          = []
+          @entry.each do |file_name|
+            files << file_name
+          end
+          assert_equal 1, files.count
+          assert_equal Pathname.new('test/files/text.txt').realpath, files.first
+        end
+
+        it 'reads paths inside of the whitelist' do
+          @entry.archive_directory = nil
+          files                    = []
+          @entry.stub(:whitelist_paths, [Pathname.new('test/files').realpath.to_s]) do
+            @entry.each do |file_name|
+              files << file_name
+            end
+          end
+          assert_equal nil, @entry.archive_directory
+          assert_equal 1, files.count
+          assert_equal Pathname.new('test/files/text.txt').realpath, files.first
+        end
+
+        it 'skips paths outside of the whitelist' do
+          @entry.archive_directory = nil
+          files                    = []
+          @entry.stub(:whitelist_paths, [Pathname.new('test/config').realpath.to_s]) do
+            @entry.each do |file_name|
+              files << file_name
+            end
+          end
+          assert_equal nil, @entry.archive_directory
+          assert_equal 0, files.count
+        end
+      end
     end
 
   end

data/test/dirmon_job_test.rb

@@ -118,11 +118,12 @@ class DirmonJobTest < Minitest::Test
       it 'skip files in archive directory' do
         @entry.archive_directory = nil
         @entry.pattern           = "#{@directory}/abc/**/*"
+        file_pathname            = Pathname.new("#{@directory}/abc/file1")
 
-        create_file("#{@directory}/abc/file1", 5)
+        create_file(file_pathname.to_s, 5)
         create_file("#{@directory}/abc/file2", 10)
-        FileUtils.makedirs("#{@directory}/abc/#{@entry.archive_pathname}")
-        create_file("#{@directory}/abc/#{@entry.archive_pathname}/file3", 10)
+        FileUtils.makedirs(@entry.archive_pathname(file_pathname))
+        create_file("#{@entry.archive_pathname(file_pathname)}/file3", 10)
 
         result = @dirmon_job.send(:check_directories, {})
 

data/test/files/_archive/archived.txt

@@ -0,0 +1,3 @@
+Hello World
+Line2
+Line3

data/test/files/text.txt

@@ -0,0 +1,3 @@
+Hello World
+Line2
+Line3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-10 00:00:00.000000000 Z
+date: 2015-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.2.0
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +135,8 @@ files:
 - test/config/mongo.yml
 - test/dirmon_entry_test.rb
 - test/dirmon_job_test.rb
+- test/files/_archive/archived.txt
+- test/files/text.txt
 - test/job_test.rb
 - test/job_worker_test.rb
 - test/jobs/test_job.rb
@@ -168,6 +170,8 @@ test_files:
 - test/config/mongo.yml
 - test/dirmon_entry_test.rb
 - test/dirmon_job_test.rb
+- test/files/_archive/archived.txt
+- test/files/text.txt
 - test/job_test.rb
 - test/job_worker_test.rb
 - test/jobs/test_job.rb