garru-distributed_logreader 0.8.0 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/distributed_logreader.gemspec +5 -2
- data/lib/distributed_logreader.rb +1 -2
- data/lib/distributed_logreader/archiver/date_dir.rb +5 -2
- data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb +7 -2
- data/lib/distributed_logreader/log_reader.rb +4 -1
- data/spec/log_reader_spec.rb +1 -1
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.10.0
|
@@ -1,12 +1,15 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
1
4
|
# -*- encoding: utf-8 -*-
|
2
5
|
|
3
6
|
Gem::Specification.new do |s|
|
4
7
|
s.name = %q{distributed_logreader}
|
5
|
-
s.version = "0.
|
8
|
+
s.version = "0.10.0"
|
6
9
|
|
7
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
11
|
s.authors = ["Gary Tsang"]
|
9
|
-
s.date = %q{2009-
|
12
|
+
s.date = %q{2009-09-14}
|
10
13
|
s.email = %q{gary@garru.com}
|
11
14
|
s.extra_rdoc_files = [
|
12
15
|
"LICENSE",
|
@@ -1,4 +1,3 @@
|
|
1
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
1
|
require 'distributed_logreader/selector.rb'
|
3
2
|
require 'distributed_logreader/achiver.rb'
|
4
3
|
require 'distributed_logreader/util.rb'
|
@@ -9,4 +8,4 @@ require 'logger'
|
|
9
8
|
|
10
9
|
$dlog_logger = Logger.new("/var/log/distributed_logreader.log")
|
11
10
|
$dlog_logger.level = Logger::DEBUG
|
12
|
-
$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
|
11
|
+
$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
|
@@ -9,8 +9,11 @@ module DLogReader
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def archive(file)
|
12
|
-
|
13
|
-
|
12
|
+
unless base_backup_dir.nil?
|
13
|
+
backup_filename = backup_dir
|
14
|
+
mv(file, backup_dir)
|
15
|
+
`bzip2 #{File.join(backup_dir, file.split('/').last)}`
|
16
|
+
end
|
14
17
|
end
|
15
18
|
|
16
19
|
protected
|
@@ -14,7 +14,11 @@ module DLogReader
|
|
14
14
|
$dlog_logger.info("Started #{log_file}:")
|
15
15
|
lines_processed = 0
|
16
16
|
@log_reader = LogReader.new(log_file) do |line|
|
17
|
-
|
17
|
+
begin
|
18
|
+
@worker.call(line)
|
19
|
+
rescue Exception => e
|
20
|
+
$dlog_logger.warn("Exception thrown in worker #{e.message}")
|
21
|
+
end
|
18
22
|
lines_processed += 1
|
19
23
|
end
|
20
24
|
@log_reader.run
|
@@ -34,13 +38,14 @@ module DLogReader
|
|
34
38
|
|
35
39
|
def post_process
|
36
40
|
self.archiver.archive(log_file) unless current?
|
41
|
+
File.delete(@log_reader.statefile) rescue nil
|
37
42
|
end
|
38
43
|
|
39
44
|
def current?
|
40
45
|
directory = File.dirname(log_file)
|
41
46
|
basename = File.basename(directory)
|
42
47
|
current_file = Dir[File.join(directory, "*")].detect{|x| x.match(/current/)}
|
43
|
-
File.exists?(current_file) && File.identical?(current_file, log_file)
|
48
|
+
!current_file.nil? && File.exists?(current_file) && File.identical?(current_file, log_file)
|
44
49
|
end
|
45
50
|
end
|
46
51
|
end
|
@@ -16,12 +16,15 @@ module DLogReader
|
|
16
16
|
load_saved_state(f)
|
17
17
|
# raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
|
18
18
|
unless f.eof?
|
19
|
+
last_report = Time.now
|
19
20
|
line_count = 0
|
20
21
|
f.each_line do |line|
|
21
22
|
@b.call(line)
|
22
23
|
line_count += 1
|
23
24
|
if (line_count % 100 == 0)
|
24
|
-
|
25
|
+
time_passed = Time.now - last_report
|
26
|
+
$dlog_logger.info( "#{Time.now.to_s} #{filename}: Processed (#{line_count}) lines in #{time_passed}s [#{(line_count.to_f / time_passed.to_f).to_i} lines/s]")
|
27
|
+
last_report = Time.now
|
25
28
|
save_state(f)
|
26
29
|
end
|
27
30
|
end
|
data/spec/log_reader_spec.rb
CHANGED
@@ -5,7 +5,7 @@ require 'fileutils'
|
|
5
5
|
describe "DLogReader::LogReader" do
|
6
6
|
before(:all) do
|
7
7
|
test_file = File.join(File.dirname(__FILE__), 'fixtures', 'test_file')
|
8
|
-
FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading'))
|
8
|
+
FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading')) rescue nil
|
9
9
|
@test_cp = File.join(File.dirname(__FILE__), 'fixtures', 'logreading', 'test')
|
10
10
|
FileUtils.cp(test_file, @test_cp)
|
11
11
|
test_fh = File.open(test_file)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: garru-distributed_logreader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gary Tsang
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
12
|
+
date: 2009-09-14 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|