garru-distributed_logreader 0.7.1 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/distributed_logreader.gemspec +2 -2
- data/lib/distributed_logreader.rb +1 -1
- data/lib/distributed_logreader/distributed_log_reader.rb +1 -1
- data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb +22 -5
- data/lib/distributed_logreader/distributer/pandemic_processor.rb +14 -8
- data/lib/distributed_logreader/distributer/simple_forked_process.rb +2 -6
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.8.0
|
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{distributed_logreader}
|
5
|
-
s.version = "0.
|
5
|
+
s.version = "0.8.0"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Gary Tsang"]
|
9
|
-
s.date = %q{2009-08-
|
9
|
+
s.date = %q{2009-08-12}
|
10
10
|
s.email = %q{gary@garru.com}
|
11
11
|
s.extra_rdoc_files = [
|
12
12
|
"LICENSE",
|
@@ -8,5 +8,5 @@ require 'distributed_logreader/distributed_log_reader/scribe_reader'
|
|
8
8
|
require 'logger'
|
9
9
|
|
10
10
|
$dlog_logger = Logger.new("/var/log/distributed_logreader.log")
|
11
|
-
$dlog_logger.level = Logger::
|
11
|
+
$dlog_logger.level = Logger::DEBUG
|
12
12
|
$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
|
@@ -11,7 +11,7 @@ module DLogReader
|
|
11
11
|
def initialize(filename, worker, num_threads = 100)
|
12
12
|
self.filename = filename
|
13
13
|
# self.distributer = SimpleForked.new(worker, 5, num_threads)
|
14
|
-
self.distributer = SimpleThreadPool.new(worker,
|
14
|
+
self.distributer = SimpleThreadPool.new(worker, num_threads)
|
15
15
|
end
|
16
16
|
|
17
17
|
# selector/archiver seem to be strongly connected. it's possible it
|
@@ -1,14 +1,31 @@
|
|
1
1
|
module DLogReader
|
2
|
-
class ScribeReader
|
3
|
-
attr_accessor :selector, :archiver
|
2
|
+
class ScribeReader
|
3
|
+
attr_accessor :selector, :archiver, :filename
|
4
4
|
attr_reader :log_reader
|
5
5
|
def initialize(filename, backupdir, worker, num_threads = 10)
|
6
|
-
|
6
|
+
self.filename = filename
|
7
7
|
self.selector = RotatingLog.new
|
8
8
|
self.selector.ignore_conditions << lambda{|x| !x.match(/scribe_stats/).nil?}
|
9
|
+
@worker = worker
|
9
10
|
self.archiver = DateDir.new(backupdir)
|
10
11
|
end
|
11
|
-
|
12
|
+
|
13
|
+
def process
|
14
|
+
$dlog_logger.info("Started #{log_file}:")
|
15
|
+
lines_processed = 0
|
16
|
+
@log_reader = LogReader.new(log_file) do |line|
|
17
|
+
@worker.call(line)
|
18
|
+
lines_processed += 1
|
19
|
+
end
|
20
|
+
@log_reader.run
|
21
|
+
$dlog_logger.info("Finished #{log_file}: Processed (#{lines_processed}) lines")
|
22
|
+
post_process
|
23
|
+
end
|
24
|
+
|
25
|
+
def log_file
|
26
|
+
self.filename
|
27
|
+
end
|
28
|
+
|
12
29
|
def log_file
|
13
30
|
@log_file ||= begin
|
14
31
|
selector.file_to_process(filename)
|
@@ -22,7 +39,7 @@ module DLogReader
|
|
22
39
|
def current?
|
23
40
|
directory = File.dirname(log_file)
|
24
41
|
basename = File.basename(directory)
|
25
|
-
current_file = File.join(directory, "
|
42
|
+
current_file = Dir[File.join(directory, "*")].detect{|x| x.match(/current/)}
|
26
43
|
File.exists?(current_file) && File.identical?(current_file, log_file)
|
27
44
|
end
|
28
45
|
end
|
@@ -15,6 +15,7 @@ module DLogReader
|
|
15
15
|
@in = read_from_child
|
16
16
|
@max_queue_size = 100
|
17
17
|
@counter = MutexCounter.new
|
18
|
+
@job_mutex = Mutex.new
|
18
19
|
wait_for_responses
|
19
20
|
else
|
20
21
|
$dlog_logger.debug("Forked")
|
@@ -43,12 +44,15 @@ module DLogReader
|
|
43
44
|
def process(body)
|
44
45
|
if parent?
|
45
46
|
while(@counter.real_total > @max_queue_size)
|
47
|
+
$dlog_logger.debug("Max process queue size: #{@counter.real_total}")
|
46
48
|
sleep(0.01)
|
47
49
|
end
|
48
50
|
body = (body.chomp + "\n")
|
49
|
-
|
50
|
-
|
51
|
-
|
51
|
+
@job_mutex.synchronize do
|
52
|
+
@out.write(body)
|
53
|
+
in_queue = @counter.inc
|
54
|
+
$dlog_logger.debug("Parent: writing #{body.inspect} - #{in_queue}")
|
55
|
+
end
|
52
56
|
else
|
53
57
|
$dlog_logger.debug("Child Processing #{body}")
|
54
58
|
return @handler.call(body)
|
@@ -76,9 +80,11 @@ module DLogReader
|
|
76
80
|
Thread.new do
|
77
81
|
loop do
|
78
82
|
line = @job_queue.pop
|
79
|
-
|
80
|
-
|
81
|
-
|
83
|
+
@job_mutex.synchronize do
|
84
|
+
process(line)
|
85
|
+
@response_queue << :a
|
86
|
+
$dlog_logger.debug("Child: Finished #{line.inspect}")
|
87
|
+
end
|
82
88
|
end
|
83
89
|
end
|
84
90
|
end
|
@@ -88,9 +94,9 @@ module DLogReader
|
|
88
94
|
loop do
|
89
95
|
ready, = IO.select([@in], nil, nil)
|
90
96
|
if ready
|
91
|
-
$dlog_logger.debug("Parent: Reading Response")
|
92
97
|
@in.readchar
|
93
|
-
@counter.decr
|
98
|
+
in_queue = @counter.decr
|
99
|
+
$dlog_logger.debug("Parent: Reading Response #{in_queue}")
|
94
100
|
end
|
95
101
|
end
|
96
102
|
end
|
@@ -3,12 +3,11 @@ require File.join(File.dirname(__FILE__), 'pandemic_processor')
|
|
3
3
|
require File.join(File.dirname(__FILE__), 'mutex_counter')
|
4
4
|
module DLogReader
|
5
5
|
class SimpleForked
|
6
|
-
attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :
|
6
|
+
attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :processors
|
7
7
|
def initialize(worker, num_processes = 3, num_threads_per_process = 10)
|
8
8
|
self.worker = worker
|
9
|
-
self.num_threads_per_process = num_threads_per_process
|
9
|
+
self.num_threads_per_process = (num_threads_per_process || 10)
|
10
10
|
self.queue = Queue.new
|
11
|
-
self.max_queue_size = 100
|
12
11
|
self.processors = []
|
13
12
|
num_processes.times do |x|
|
14
13
|
$dlog_logger.debug("Forking #{x} process")
|
@@ -17,9 +16,6 @@ module DLogReader
|
|
17
16
|
end
|
18
17
|
|
19
18
|
def process(line)
|
20
|
-
while(queue.size > self.max_queue_size)
|
21
|
-
sleep(0.1)
|
22
|
-
end
|
23
19
|
self.queue << line
|
24
20
|
end
|
25
21
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: garru-distributed_logreader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gary Tsang
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-08-
|
12
|
+
date: 2009-08-12 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|