file_worker 0.0.1 → 0.1.0

data/bin/file_worker

@@ -1,17 +1,13 @@
-#!/usr/bin/env ruby
+#!/usr/bin/env jruby
 
-require "rubygems"
-require "bundler"
-Bundler.require
-
-require 'file_worker/cli'
+require 'file_worker'
 
 scanner = FileWorker::Cli.build_scanner
 
 scanner.on_error do |file_name, exception|
   $stderr.puts("Failed to process #{file_name}")
-  $stderr.puts(ex)
-  $stderr.puts(ex.backtrace.join("\n"))
+  $stderr.puts(exception)
+  $stderr.puts(exception.backtrace.join("\n"))
 end
 
 scanner.start

data/lib/file_worker/cli.rb

@@ -40,8 +40,6 @@ module FileWorker
 
       options[:out_directory] ||= File.expand_path('../done', options[:in_directory])
 
-      puts options.inspect
-
       DirectoryScanner.new(options)
     end
   end

data/lib/file_worker/directory_scanner.rb

@@ -12,7 +12,9 @@ module FileWorker
       @options   = options
       @in_path   = Pathname.new(@options[:in_directory])
       @done_path = Pathname.new(@options[:out_directory])
+      @glob_path = @in_path + '**/*'
       @sleep     = @options[:sleep] || 1
+      @workers   = @options[:workers] || 5
 
       @max_queue_size = @options[:max_queue_size] || 1000
 
@@ -31,7 +33,10 @@ module FileWorker
 
       begin
         @worker_class.new(file_name, @options).process
-        FileUtils.mv(file_name, @done_path)
+
+        done_file_name = file_name.sub(@in_path, @done_path)
+        FileUtils.mkdir_p(File.dirname(done_file_name))
+        FileUtils.mv(file_name, done_file_name)
       rescue Exception => e
         handle_error(file_name, e)
       end
@@ -40,7 +45,7 @@ module FileWorker
     end
 
     def queue
-      @queue ||= GirlFriday::WorkQueue.new(@queue_name, :size => 3) do |file_name|
+      @queue ||= GirlFriday::WorkQueue.new(@queue_name, :size => @workers) do |file_name|
         process(file_name)
       end
     end
@@ -66,7 +71,8 @@ module FileWorker
     end
 
     def scan
-      file_names = Dir.glob(@in_path + '*') - @state.keys
+      file_names = Dir.glob(@glob_path) - @state.keys
+      file_names.reject! { |file_name| File.directory?(file_name) }
 
       max_items = @max_queue_size - queue_size
 
@@ -76,6 +82,15 @@ module FileWorker
     end
 
     def start
+      puts "File Worker Starting"
+      puts "  in directory    #{@in_path}"
+      puts "  out directory   #{@done_path}"
+      puts "  workers         #{@workers}"
+      puts "  max queue size  #{@max_queue_size}"
+      puts "  scan interval   #{@sleep}s"
+      puts "  worker          #{@worker_class.name}"
+      puts ""
+
       @run = true
       while @run
         scan

data/lib/file_worker/version.rb

@@ -1,3 +1,3 @@
 module FileWorker
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/file_worker.rb

@@ -4,4 +4,5 @@ module FileWorker
   autoload :DirectoryScanner, 'file_worker/directory_scanner'
   autoload :DefaultWorker,    'file_worker/default_worker'
   autoload :S3UploadWorker,   'file_worker/s3_upload_worker'
+  autoload :Cli,              'file_worker/cli'
 end

metadata

@@ -2,7 +2,7 @@
 name: file_worker
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors: 
   - Mick Staugaard