file_worker 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/file_worker +4 -8
- data/lib/file_worker/cli.rb +0 -2
- data/lib/file_worker/directory_scanner.rb +18 -3
- data/lib/file_worker/version.rb +1 -1
- data/lib/file_worker.rb +1 -0
- metadata +1 -1
data/bin/file_worker
CHANGED
@@ -1,17 +1,13 @@
|
|
1
|
-
#!/usr/bin/env
|
1
|
+
#!/usr/bin/env jruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require "bundler"
|
5
|
-
Bundler.require
|
6
|
-
|
7
|
-
require 'file_worker/cli'
|
3
|
+
require 'file_worker'
|
8
4
|
|
9
5
|
scanner = FileWorker::Cli.build_scanner
|
10
6
|
|
11
7
|
scanner.on_error do |file_name, exception|
|
12
8
|
$stderr.puts("Failed to process #{file_name}")
|
13
|
-
$stderr.puts(
|
14
|
-
$stderr.puts(
|
9
|
+
$stderr.puts(exception)
|
10
|
+
$stderr.puts(exception.backtrace.join("\n"))
|
15
11
|
end
|
16
12
|
|
17
13
|
scanner.start
|
data/lib/file_worker/cli.rb
CHANGED
@@ -12,7 +12,9 @@ module FileWorker
|
|
12
12
|
@options = options
|
13
13
|
@in_path = Pathname.new(@options[:in_directory])
|
14
14
|
@done_path = Pathname.new(@options[:out_directory])
|
15
|
+
@glob_path = @in_path + '**/*'
|
15
16
|
@sleep = @options[:sleep] || 1
|
17
|
+
@workers = @options[:workers] || 5
|
16
18
|
|
17
19
|
@max_queue_size = @options[:max_queue_size] || 1000
|
18
20
|
|
@@ -31,7 +33,10 @@ module FileWorker
|
|
31
33
|
|
32
34
|
begin
|
33
35
|
@worker_class.new(file_name, @options).process
|
34
|
-
|
36
|
+
|
37
|
+
done_file_name = file_name.sub(@in_path, @done_path)
|
38
|
+
FileUtils.mkdir_p(File.dirname(done_file_name))
|
39
|
+
FileUtils.mv(file_name, done_file_name)
|
35
40
|
rescue Exception => e
|
36
41
|
handle_error(file_name, e)
|
37
42
|
end
|
@@ -40,7 +45,7 @@ module FileWorker
|
|
40
45
|
end
|
41
46
|
|
42
47
|
def queue
|
43
|
-
@queue ||= GirlFriday::WorkQueue.new(@queue_name, :size =>
|
48
|
+
@queue ||= GirlFriday::WorkQueue.new(@queue_name, :size => @workers) do |file_name|
|
44
49
|
process(file_name)
|
45
50
|
end
|
46
51
|
end
|
@@ -66,7 +71,8 @@ module FileWorker
|
|
66
71
|
end
|
67
72
|
|
68
73
|
def scan
|
69
|
-
file_names = Dir.glob(@
|
74
|
+
file_names = Dir.glob(@glob_path) - @state.keys
|
75
|
+
file_names.reject! { |file_name| File.directory?(file_name) }
|
70
76
|
|
71
77
|
max_items = @max_queue_size - queue_size
|
72
78
|
|
@@ -76,6 +82,15 @@ module FileWorker
|
|
76
82
|
end
|
77
83
|
|
78
84
|
def start
|
85
|
+
puts "File Worker Starting"
|
86
|
+
puts " in directory #{@in_path}"
|
87
|
+
puts " out directory #{@done_path}"
|
88
|
+
puts " workers #{@workers}"
|
89
|
+
puts " max queue size #{@max_queue_size}"
|
90
|
+
puts " scan interval #{@sleep}s"
|
91
|
+
puts " worker #{@worker_class.name}"
|
92
|
+
puts ""
|
93
|
+
|
79
94
|
@run = true
|
80
95
|
while @run
|
81
96
|
scan
|
data/lib/file_worker/version.rb
CHANGED
data/lib/file_worker.rb
CHANGED