garru-distributed_logreader 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +7 -0
- data/LICENSE +20 -0
- data/README.rdoc +7 -0
- data/Rakefile +48 -0
- data/VERSION +1 -0
- data/distributed_logreader.gemspec +80 -0
- data/lib/distributed_logreader.rb +11 -0
- data/lib/distributed_logreader/achiver.rb +10 -0
- data/lib/distributed_logreader/archiver/date_dir.rb +33 -0
- data/lib/distributed_logreader/distributed_log_reader.rb +39 -0
- data/lib/distributed_logreader/distributed_log_reader/rotater_reader.rb +21 -0
- data/lib/distributed_logreader/distributer.rb +13 -0
- data/lib/distributed_logreader/distributer/simple_thread_pool.rb +36 -0
- data/lib/distributed_logreader/log_reader.rb +56 -0
- data/lib/distributed_logreader/selector.rb +12 -0
- data/lib/distributed_logreader/selector/rotating_log.rb +31 -0
- data/lib/distributed_logreader/util.rb +7 -0
- data/spec/archiver/date_dir_spec.rb +25 -0
- data/spec/archiver_spec.rb +13 -0
- data/spec/distributed_log_reader/rotater_reader_spec.rb +26 -0
- data/spec/distributed_log_reader_spec.rb +20 -0
- data/spec/distributer/simple_thread_pool_spec.rb +17 -0
- data/spec/distributer_spec.rb +13 -0
- data/spec/fixtures/copytruncate/test +0 -0
- data/spec/fixtures/copytruncate/test.1 +0 -0
- data/spec/fixtures/copytruncate/test_current +0 -0
- data/spec/fixtures/logrotate/test-20090101 +0 -0
- data/spec/fixtures/logrotate/test-20090102 +0 -0
- data/spec/fixtures/symlink/test +0 -0
- data/spec/fixtures/symlink/test_older_sym +0 -0
- data/spec/fixtures/test_file +4 -0
- data/spec/log_reader_spec.rb +57 -0
- data/spec/selector/rotating_log_spec.rb +24 -0
- data/spec/selector_spec.rb +13 -0
- data/spec/spec_helper.rb +9 -0
- metadata +98 -0
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Gary Tsang
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "distributed_logreader"
|
8
|
+
gem.summary = %Q{TODO}
|
9
|
+
gem.email = "gary@garru.com"
|
10
|
+
gem.homepage = "http://github.com/garru/distributed_logreader"
|
11
|
+
gem.authors = ["Gary Tsang"]
|
12
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
13
|
+
end
|
14
|
+
|
15
|
+
rescue LoadError
|
16
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'spec/rake/spectask'
|
20
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
21
|
+
spec.libs << 'lib' << 'spec'
|
22
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
23
|
+
end
|
24
|
+
|
25
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
26
|
+
spec.libs << 'lib' << 'spec'
|
27
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
28
|
+
spec.rcov = true
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
task :default => :spec
|
33
|
+
|
34
|
+
require 'rake/rdoctask'
|
35
|
+
Rake::RDocTask.new do |rdoc|
|
36
|
+
if File.exist?('VERSION.yml')
|
37
|
+
config = YAML.load(File.read('VERSION.yml'))
|
38
|
+
version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
|
39
|
+
else
|
40
|
+
version = ""
|
41
|
+
end
|
42
|
+
|
43
|
+
rdoc.rdoc_dir = 'rdoc'
|
44
|
+
rdoc.title = "distributed_logreader #{version}"
|
45
|
+
rdoc.rdoc_files.include('README*')
|
46
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
47
|
+
end
|
48
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.2.0
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{distributed_logreader}
|
5
|
+
s.version = "0.2.0"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Gary Tsang"]
|
9
|
+
s.date = %q{2009-07-30}
|
10
|
+
s.email = %q{gary@garru.com}
|
11
|
+
s.extra_rdoc_files = [
|
12
|
+
"LICENSE",
|
13
|
+
"README.rdoc"
|
14
|
+
]
|
15
|
+
s.files = [
|
16
|
+
".gitignore",
|
17
|
+
"LICENSE",
|
18
|
+
"README.rdoc",
|
19
|
+
"Rakefile",
|
20
|
+
"VERSION",
|
21
|
+
"distributed_logreader.gemspec",
|
22
|
+
"lib/distributed_logreader.rb",
|
23
|
+
"lib/distributed_logreader/achiver.rb",
|
24
|
+
"lib/distributed_logreader/archiver/date_dir.rb",
|
25
|
+
"lib/distributed_logreader/distributed_log_reader.rb",
|
26
|
+
"lib/distributed_logreader/distributed_log_reader/rotater_reader.rb",
|
27
|
+
"lib/distributed_logreader/distributer.rb",
|
28
|
+
"lib/distributed_logreader/distributer/simple_thread_pool.rb",
|
29
|
+
"lib/distributed_logreader/log_reader.rb",
|
30
|
+
"lib/distributed_logreader/selector.rb",
|
31
|
+
"lib/distributed_logreader/selector/rotating_log.rb",
|
32
|
+
"lib/distributed_logreader/util.rb",
|
33
|
+
"spec/archiver/date_dir_spec.rb",
|
34
|
+
"spec/archiver_spec.rb",
|
35
|
+
"spec/distributed_log_reader/rotater_reader_spec.rb",
|
36
|
+
"spec/distributed_log_reader_spec.rb",
|
37
|
+
"spec/distributer/simple_thread_pool_spec.rb",
|
38
|
+
"spec/distributer_spec.rb",
|
39
|
+
"spec/fixtures/copytruncate/test",
|
40
|
+
"spec/fixtures/copytruncate/test.1",
|
41
|
+
"spec/fixtures/copytruncate/test_current",
|
42
|
+
"spec/fixtures/logrotate/test-20090101",
|
43
|
+
"spec/fixtures/logrotate/test-20090102",
|
44
|
+
"spec/fixtures/symlink/test",
|
45
|
+
"spec/fixtures/symlink/test_older_sym",
|
46
|
+
"spec/fixtures/test_file",
|
47
|
+
"spec/log_reader_spec.rb",
|
48
|
+
"spec/selector/rotating_log_spec.rb",
|
49
|
+
"spec/selector_spec.rb",
|
50
|
+
"spec/spec_helper.rb"
|
51
|
+
]
|
52
|
+
s.has_rdoc = true
|
53
|
+
s.homepage = %q{http://github.com/garru/distributed_logreader}
|
54
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
55
|
+
s.require_paths = ["lib"]
|
56
|
+
s.rubygems_version = %q{1.3.1}
|
57
|
+
s.summary = %q{TODO}
|
58
|
+
s.test_files = [
|
59
|
+
"spec/archiver/date_dir_spec.rb",
|
60
|
+
"spec/archiver_spec.rb",
|
61
|
+
"spec/distributed_log_reader/rotater_reader_spec.rb",
|
62
|
+
"spec/distributed_log_reader_spec.rb",
|
63
|
+
"spec/distributer/simple_thread_pool_spec.rb",
|
64
|
+
"spec/distributer_spec.rb",
|
65
|
+
"spec/log_reader_spec.rb",
|
66
|
+
"spec/selector/rotating_log_spec.rb",
|
67
|
+
"spec/selector_spec.rb",
|
68
|
+
"spec/spec_helper.rb"
|
69
|
+
]
|
70
|
+
|
71
|
+
if s.respond_to? :specification_version then
|
72
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
73
|
+
s.specification_version = 2
|
74
|
+
|
75
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
76
|
+
else
|
77
|
+
end
|
78
|
+
else
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
require 'distributed_logreader/selector.rb'
|
3
|
+
require 'distributed_logreader/achiver.rb'
|
4
|
+
require 'distributed_logreader/util.rb'
|
5
|
+
require 'distributed_logreader/distributed_log_reader'
|
6
|
+
require 'distributed_logreader/distributed_log_reader/rotater_reader'
|
7
|
+
require 'logger'
|
8
|
+
|
9
|
+
$dlog_logger = Logger.new("distributed_logreader.log")
|
10
|
+
$dlog_logger.level = Logger::INFO
|
11
|
+
$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# This abstract class defines the interface that handles log file archiving
|
2
|
+
|
3
|
+
module DLogReader
|
4
|
+
class Archiver
|
5
|
+
# archive file as you see fit
|
6
|
+
def archive(filename)
|
7
|
+
raise NotImplementedError.new("archive not implemented. Are you sure you created a conrete class?")
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module DLogReader
|
3
|
+
class DateDir < Archiver
|
4
|
+
include FileUtils
|
5
|
+
attr_accessor :base_backup_dir
|
6
|
+
|
7
|
+
def initialize(backup_dir)
|
8
|
+
self.base_backup_dir = backup_dir
|
9
|
+
end
|
10
|
+
|
11
|
+
def backup(file)
|
12
|
+
mv(file, backup_dir) unless base_backup_dir.nil?
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
protected
|
17
|
+
|
18
|
+
def backup_dir
|
19
|
+
time = Time.now
|
20
|
+
directory_structure = [base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s]
|
21
|
+
temp_dir = []
|
22
|
+
directory_structure.each do |x|
|
23
|
+
temp_dir << x
|
24
|
+
temp_file = File.join(temp_dir)
|
25
|
+
unless File.exist?(temp_file)
|
26
|
+
mkdir(temp_file)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
File.join(temp_dir)
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'distributer', 'simple_thread_pool')
|
2
|
+
require File.join(File.dirname(__FILE__), 'selector', 'rotating_log')
|
3
|
+
require File.join(File.dirname(__FILE__), 'archiver', 'date_dir')
|
4
|
+
require File.join(File.dirname(__FILE__), 'log_reader')
|
5
|
+
|
6
|
+
module DLogReader
|
7
|
+
class DistributedLogReader
|
8
|
+
attr_accessor :distributer, :filename
|
9
|
+
attr_reader :log_reader
|
10
|
+
def initialize(filename, worker, num_threads = 10)
|
11
|
+
self.filename = filename
|
12
|
+
self.distributer = SimpleThreadPool.new(worker, num_threads)
|
13
|
+
end
|
14
|
+
|
15
|
+
# selector/archiver seem to be strongly connected. it's possible it
|
16
|
+
# needs to be moved into LogReader
|
17
|
+
def process
|
18
|
+
pre_process
|
19
|
+
@log_reader = LogReader.new(log_file) do |line|
|
20
|
+
self.distributer.process(line)
|
21
|
+
end
|
22
|
+
@log_reader.run
|
23
|
+
self.distributer.join
|
24
|
+
post_process
|
25
|
+
end
|
26
|
+
|
27
|
+
def log_file
|
28
|
+
self.filename
|
29
|
+
end
|
30
|
+
|
31
|
+
#predefined hooks
|
32
|
+
def pre_process
|
33
|
+
end
|
34
|
+
|
35
|
+
#predefined hooks
|
36
|
+
def post_process
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module DLogReader
|
2
|
+
class RotaterReader < DistributedLogReader
|
3
|
+
attr_accessor :selector, :archiver
|
4
|
+
attr_reader :log_reader
|
5
|
+
def initialize(filename, backupdir, worker, num_threads = 10)
|
6
|
+
super(filename, worker, num_threads)
|
7
|
+
self.selector = RotatingLog.new
|
8
|
+
self.archiver = DateDir.new(backupdir)
|
9
|
+
end
|
10
|
+
|
11
|
+
def log_file
|
12
|
+
@log_file ||= begin
|
13
|
+
selector.file_to_process(filename)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def post_process
|
18
|
+
self.archiver.backup(log_file)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module DLogReader
|
4
|
+
class SimpleThreadPool
|
5
|
+
attr_accessor :num_threads, :worker, :thread_pool, :queue
|
6
|
+
|
7
|
+
def initialize(worker, num_threads = 10)
|
8
|
+
self.worker = worker
|
9
|
+
self.num_threads = num_threads
|
10
|
+
self.queue = Queue.new
|
11
|
+
num_threads.times do
|
12
|
+
create_thread
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def process(line)
|
17
|
+
self.queue << line
|
18
|
+
end
|
19
|
+
|
20
|
+
def join
|
21
|
+
while(queue.size > 0)
|
22
|
+
sleep 0.1
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
def create_thread
|
28
|
+
Thread.new do
|
29
|
+
loop do
|
30
|
+
line = self.queue.pop
|
31
|
+
self.worker.call(line)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
module DLogReader
|
3
|
+
class LogReader
|
4
|
+
attr_accessor :filename
|
5
|
+
attr_writer :statefile
|
6
|
+
|
7
|
+
def initialize(filename, &b)
|
8
|
+
self.filename = filename
|
9
|
+
@b = b
|
10
|
+
end
|
11
|
+
|
12
|
+
def run
|
13
|
+
raise IOError.new("no file given") if filename.nil?
|
14
|
+
raise IOError.new("File not readable") unless File.readable?(filename)
|
15
|
+
f = File.open(filename, "r+")
|
16
|
+
load_saved_state(f)
|
17
|
+
raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
|
18
|
+
f.each_line do |line|
|
19
|
+
@b.call(line)
|
20
|
+
end
|
21
|
+
save_state(f)
|
22
|
+
f.flock(File::LOCK_UN)
|
23
|
+
end
|
24
|
+
|
25
|
+
def statefile
|
26
|
+
@statefile ||= begin
|
27
|
+
log_basename = File.basename(filename)
|
28
|
+
File.join("/tmp", "log_state_#{log_basename}")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def load_saved_state(log_filehandle)
|
35
|
+
return unless File.exists?(statefile) && !(state = File.read(statefile)).nil?
|
36
|
+
pos, l_digest = Marshal.load(state)
|
37
|
+
return if File.size(log_filehandle) < pos
|
38
|
+
log_filehandle.pos = pos if digest(log_filehandle, pos) == l_digest
|
39
|
+
end
|
40
|
+
|
41
|
+
def save_state(log_filehandle)
|
42
|
+
File.open(statefile, "w") do |f|
|
43
|
+
f.write(Marshal.dump([log_filehandle.pos, digest(log_filehandle, log_filehandle.pos)]))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def digest(log_filehandle, position)
|
48
|
+
log_filehandle.pos = 0
|
49
|
+
read_length = [position, 50].min
|
50
|
+
l = log_filehandle.read(read_length)
|
51
|
+
f_digest = Digest::MD5.hexdigest(l)
|
52
|
+
log_filehandle.pos = position
|
53
|
+
f_digest
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module DLogReader
|
2
|
+
# This abstract class defines the interface to decide which log
|
3
|
+
# file to read. The identity strategy is the simplist, to return the file
|
4
|
+
# inputed. However, to handle rotating log files, we'll need some more complex
|
5
|
+
# strategies.
|
6
|
+
class Selector
|
7
|
+
# determines the file to process from file path input
|
8
|
+
def file_to_process(file_or_dir)
|
9
|
+
raise NotImplementedError.new, "file_to_process not implemented. Are you sure you created a conrete class?"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module DLogReader
|
2
|
+
# This class chooses the oldest log file in the directory that matches the
|
3
|
+
# input filename. This should work with a variety of log rotating schemes:
|
4
|
+
# including copytruncate and date suffix.
|
5
|
+
class RotatingLog < Selector
|
6
|
+
|
7
|
+
def file_to_process(file_or_dir)
|
8
|
+
if File.directory?(file_or_dir)
|
9
|
+
directory = file_or_dir
|
10
|
+
basename = '/'
|
11
|
+
else
|
12
|
+
directory = File.dirname(file_or_dir)
|
13
|
+
basename = File.basename(file_or_dir)
|
14
|
+
end
|
15
|
+
oldest_logfile(directory, basename)
|
16
|
+
end
|
17
|
+
|
18
|
+
protected
|
19
|
+
|
20
|
+
def oldest_logfile(directory, basename)
|
21
|
+
file_list = Dir[File.join(directory, "#{basename}*")]
|
22
|
+
file_list.reject!{|x| symlink_file_in_dir?(x)}
|
23
|
+
file = file_list.size > 0 ? file_list.sort_by{|a| File.new(a).mtime}.first : nil
|
24
|
+
end
|
25
|
+
|
26
|
+
# returns true if filename is a symlink and its referring to a file already inside the current directory
|
27
|
+
def symlink_file_in_dir?(filename)
|
28
|
+
File.symlink?(filename) && File.dirname(File.readlink(filename)) == File.dirname(filename)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'distributed_logreader/archiver/date_dir'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
describe "DLogReader::DateDir" do
|
6
|
+
before(:all) do
|
7
|
+
FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
|
8
|
+
@file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
|
9
|
+
@base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
|
10
|
+
time = Time.now
|
11
|
+
@backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
|
12
|
+
@archiver = DLogReader::DateDir.new(@base_backup_dir)
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "backup" do
|
16
|
+
it "should move file into Y/M/D backup directory" do
|
17
|
+
@archiver.backup(@file_path)
|
18
|
+
File.exist?(@backup_dir).should == true
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
after(:all) do
|
23
|
+
FileUtils.rm_r(@base_backup_dir)
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe "DLogReader::LogArchiver" do
|
4
|
+
before(:all) do
|
5
|
+
@archiver = DLogReader::Archiver.new
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "archive" do
|
9
|
+
it "should raise NotImplementedError" do
|
10
|
+
lambda{ @archiver.archive('dummy_file') }.should raise_error(NotImplementedError)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
describe "DLogReader::RotaterLogreader" do
|
5
|
+
before(:all) do
|
6
|
+
FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
|
7
|
+
@file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
|
8
|
+
@base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
|
9
|
+
time = Time.now
|
10
|
+
@backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
|
11
|
+
@logreader = DLogReader::RotaterReader.new(@file_path, @base_backup_dir, lambda{|x| puts x})
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "process" do
|
15
|
+
it 'should' do
|
16
|
+
@logreader.process
|
17
|
+
File.exist?(@backup_dir).should == true
|
18
|
+
File.exist?(@file_path).should == false
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
after(:all) do
|
23
|
+
FileUtils.rm_r(@base_backup_dir)
|
24
|
+
FileUtils.rm_r(@logreader.log_reader.statefile)
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
describe "DLogReader::DistributedLogreader" do
|
5
|
+
before(:all) do
|
6
|
+
FileUtils.cp(File.join(File.dirname(__FILE__), 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), 'fixtures', 'test_file2'))
|
7
|
+
@file_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_file2')
|
8
|
+
@logreader = DLogReader::DistributedLogReader.new(@file_path, lambda{|x| puts x})
|
9
|
+
end
|
10
|
+
|
11
|
+
describe "process" do
|
12
|
+
it 'should' do
|
13
|
+
@logreader.process
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
after(:all) do
|
18
|
+
FileUtils.rm_r(@logreader.log_reader.statefile)
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'distributed_logreader/distributer/simple_thread_pool'
|
3
|
+
|
4
|
+
describe "DLogReader::SimpleThreadPool" do
|
5
|
+
before(:all) do
|
6
|
+
@thread_pool = DLogReader::SimpleThreadPool.new(lambda{|x| x}, 10)
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "process" do
|
10
|
+
it 'should process with lots of threads' do
|
11
|
+
100.times do |x|
|
12
|
+
@thread_pool.process(x.to_s)
|
13
|
+
end
|
14
|
+
@thread_pool.join
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'distributed_logreader/distributer'
|
3
|
+
|
4
|
+
describe "DLogReader::Distributer" do
|
5
|
+
before(:all) do
|
6
|
+
@distributer = DLogReader::Distributer.new(lambda{|x| x})
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "process" do
|
10
|
+
it "should raise NotImplementedError" do
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'distributed_logreader/log_reader'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
describe "DLogReader::LogReader" do
|
6
|
+
before(:all) do
|
7
|
+
test_file = File.join(File.dirname(__FILE__), 'fixtures', 'test_file')
|
8
|
+
FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading'))
|
9
|
+
@test_cp = File.join(File.dirname(__FILE__), 'fixtures', 'logreading', 'test')
|
10
|
+
FileUtils.cp(test_file, @test_cp)
|
11
|
+
test_fh = File.open(test_file)
|
12
|
+
|
13
|
+
@reader = DLogReader::LogReader.new(@test_cp) do |line|
|
14
|
+
unless test_fh.readline == line
|
15
|
+
raise RuntimeError.new, 'you messed up bud'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
@test_line = "this is an added line. this should be read first\n"
|
19
|
+
@state_writer = DLogReader::LogReader.new(@test_cp){|line| line;}
|
20
|
+
@state_reader = DLogReader::LogReader.new(@test_cp) do |line|
|
21
|
+
unless line == @test_line
|
22
|
+
raise RuntimeError.new, 'you messed up worse'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "run" do
|
28
|
+
it 'should read log files' do
|
29
|
+
lambda{@reader.run}.should_not raise_error
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'should resume from last access' do
|
33
|
+
#lets read to the end of file and write state
|
34
|
+
lambda{@state_writer.run}.should_not raise_error
|
35
|
+
fh = File.open(@test_cp, 'a')
|
36
|
+
fh.write(@test_line)
|
37
|
+
fh.close
|
38
|
+
lambda{@state_reader.run}.should_not raise_error
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should detect if log is different from last and to start from beg of file' do
|
42
|
+
lambda{@state_writer.run}.should_not raise_error
|
43
|
+
fh = File.open(@test_cp, 'w')
|
44
|
+
fh.write(@test_line)
|
45
|
+
fh.close
|
46
|
+
lambda{@state_reader.run}.should_not raise_error
|
47
|
+
fh = File.open(@test_cp, 'w')
|
48
|
+
fh.write('')
|
49
|
+
fh.close
|
50
|
+
lambda{@state_reader.run}.should_not raise_error
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
after(:all) do
|
55
|
+
FileUtils.rm_r(File.dirname(@test_cp))
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'distributed_logreader/selector/rotating_log'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
describe "DLogReader::RotatingLog" do
|
6
|
+
before(:all) do
|
7
|
+
@chooser = DLogReader::RotatingLog.new
|
8
|
+
end
|
9
|
+
|
10
|
+
describe "file_to_process" do
|
11
|
+
it "should pick the oldest file for logs in copytruncate format (file)" do
|
12
|
+
@chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test.1')
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should pick the oldest file in timestamp suffix log format (dirname)" do
|
16
|
+
@chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate', 'test-20090101')
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should pick the oldest file ignoring symlinks pointing to files already in dir" do
|
20
|
+
@chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink', 'test')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe "DLogReader::Selector" do
|
4
|
+
before(:all) do
|
5
|
+
@chooser = DLogReader::Selector.new
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "file_to_process" do
|
9
|
+
it "should raise NotImplementedError" do
|
10
|
+
lambda{ @chooser.file_to_process('dummy_file') }.should raise_error(NotImplementedError)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: garru-distributed_logreader
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Gary Tsang
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-07-30 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description:
|
17
|
+
email: gary@garru.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files:
|
23
|
+
- LICENSE
|
24
|
+
- README.rdoc
|
25
|
+
files:
|
26
|
+
- .gitignore
|
27
|
+
- LICENSE
|
28
|
+
- README.rdoc
|
29
|
+
- Rakefile
|
30
|
+
- VERSION
|
31
|
+
- distributed_logreader.gemspec
|
32
|
+
- lib/distributed_logreader.rb
|
33
|
+
- lib/distributed_logreader/achiver.rb
|
34
|
+
- lib/distributed_logreader/archiver/date_dir.rb
|
35
|
+
- lib/distributed_logreader/distributed_log_reader.rb
|
36
|
+
- lib/distributed_logreader/distributed_log_reader/rotater_reader.rb
|
37
|
+
- lib/distributed_logreader/distributer.rb
|
38
|
+
- lib/distributed_logreader/distributer/simple_thread_pool.rb
|
39
|
+
- lib/distributed_logreader/log_reader.rb
|
40
|
+
- lib/distributed_logreader/selector.rb
|
41
|
+
- lib/distributed_logreader/selector/rotating_log.rb
|
42
|
+
- lib/distributed_logreader/util.rb
|
43
|
+
- spec/archiver/date_dir_spec.rb
|
44
|
+
- spec/archiver_spec.rb
|
45
|
+
- spec/distributed_log_reader/rotater_reader_spec.rb
|
46
|
+
- spec/distributed_log_reader_spec.rb
|
47
|
+
- spec/distributer/simple_thread_pool_spec.rb
|
48
|
+
- spec/distributer_spec.rb
|
49
|
+
- spec/fixtures/copytruncate/test
|
50
|
+
- spec/fixtures/copytruncate/test.1
|
51
|
+
- spec/fixtures/copytruncate/test_current
|
52
|
+
- spec/fixtures/logrotate/test-20090101
|
53
|
+
- spec/fixtures/logrotate/test-20090102
|
54
|
+
- spec/fixtures/symlink/test
|
55
|
+
- spec/fixtures/symlink/test_older_sym
|
56
|
+
- spec/fixtures/test_file
|
57
|
+
- spec/log_reader_spec.rb
|
58
|
+
- spec/selector/rotating_log_spec.rb
|
59
|
+
- spec/selector_spec.rb
|
60
|
+
- spec/spec_helper.rb
|
61
|
+
has_rdoc: true
|
62
|
+
homepage: http://github.com/garru/distributed_logreader
|
63
|
+
licenses:
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options:
|
66
|
+
- --charset=UTF-8
|
67
|
+
require_paths:
|
68
|
+
- lib
|
69
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: "0"
|
74
|
+
version:
|
75
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: "0"
|
80
|
+
version:
|
81
|
+
requirements: []
|
82
|
+
|
83
|
+
rubyforge_project:
|
84
|
+
rubygems_version: 1.3.5
|
85
|
+
signing_key:
|
86
|
+
specification_version: 2
|
87
|
+
summary: TODO
|
88
|
+
test_files:
|
89
|
+
- spec/archiver/date_dir_spec.rb
|
90
|
+
- spec/archiver_spec.rb
|
91
|
+
- spec/distributed_log_reader/rotater_reader_spec.rb
|
92
|
+
- spec/distributed_log_reader_spec.rb
|
93
|
+
- spec/distributer/simple_thread_pool_spec.rb
|
94
|
+
- spec/distributer_spec.rb
|
95
|
+
- spec/log_reader_spec.rb
|
96
|
+
- spec/selector/rotating_log_spec.rb
|
97
|
+
- spec/selector_spec.rb
|
98
|
+
- spec/spec_helper.rb
|