garru-distributed_logreader 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. data/.gitignore +7 -0
  2. data/LICENSE +20 -0
  3. data/README.rdoc +7 -0
  4. data/Rakefile +48 -0
  5. data/VERSION +1 -0
  6. data/distributed_logreader.gemspec +80 -0
  7. data/lib/distributed_logreader.rb +11 -0
  8. data/lib/distributed_logreader/achiver.rb +10 -0
  9. data/lib/distributed_logreader/archiver/date_dir.rb +33 -0
  10. data/lib/distributed_logreader/distributed_log_reader.rb +39 -0
  11. data/lib/distributed_logreader/distributed_log_reader/rotater_reader.rb +21 -0
  12. data/lib/distributed_logreader/distributer.rb +13 -0
  13. data/lib/distributed_logreader/distributer/simple_thread_pool.rb +36 -0
  14. data/lib/distributed_logreader/log_reader.rb +56 -0
  15. data/lib/distributed_logreader/selector.rb +12 -0
  16. data/lib/distributed_logreader/selector/rotating_log.rb +31 -0
  17. data/lib/distributed_logreader/util.rb +7 -0
  18. data/spec/archiver/date_dir_spec.rb +25 -0
  19. data/spec/archiver_spec.rb +13 -0
  20. data/spec/distributed_log_reader/rotater_reader_spec.rb +26 -0
  21. data/spec/distributed_log_reader_spec.rb +20 -0
  22. data/spec/distributer/simple_thread_pool_spec.rb +17 -0
  23. data/spec/distributer_spec.rb +13 -0
  24. data/spec/fixtures/copytruncate/test +0 -0
  25. data/spec/fixtures/copytruncate/test.1 +0 -0
  26. data/spec/fixtures/copytruncate/test_current +0 -0
  27. data/spec/fixtures/logrotate/test-20090101 +0 -0
  28. data/spec/fixtures/logrotate/test-20090102 +0 -0
  29. data/spec/fixtures/symlink/test +0 -0
  30. data/spec/fixtures/symlink/test_older_sym +0 -0
  31. data/spec/fixtures/test_file +4 -0
  32. data/spec/log_reader_spec.rb +57 -0
  33. data/spec/selector/rotating_log_spec.rb +24 -0
  34. data/spec/selector_spec.rb +13 -0
  35. data/spec/spec_helper.rb +9 -0
  36. metadata +98 -0
data/.gitignore ADDED
@@ -0,0 +1,7 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
6
+ *log
7
+ temp_backup_dir/*
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2009 Gary Tsang
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.rdoc ADDED
@@ -0,0 +1,7 @@
1
+ = distributed_logreader
2
+
3
+ Description goes here.
4
+
5
+ == Copyright
6
+
7
+ Copyright (c) 2009 Gary Tsang. See LICENSE for details.
data/Rakefile ADDED
@@ -0,0 +1,48 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "distributed_logreader"
8
+ gem.summary = %Q{TODO}
9
+ gem.email = "gary@garru.com"
10
+ gem.homepage = "http://github.com/garru/distributed_logreader"
11
+ gem.authors = ["Gary Tsang"]
12
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
13
+ end
14
+
15
+ rescue LoadError
16
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
17
+ end
18
+
19
+ require 'spec/rake/spectask'
20
+ Spec::Rake::SpecTask.new(:spec) do |spec|
21
+ spec.libs << 'lib' << 'spec'
22
+ spec.spec_files = FileList['spec/**/*_spec.rb']
23
+ end
24
+
25
+ Spec::Rake::SpecTask.new(:rcov) do |spec|
26
+ spec.libs << 'lib' << 'spec'
27
+ spec.pattern = 'spec/**/*_spec.rb'
28
+ spec.rcov = true
29
+ end
30
+
31
+
32
+ task :default => :spec
33
+
34
+ require 'rake/rdoctask'
35
+ Rake::RDocTask.new do |rdoc|
36
+ if File.exist?('VERSION.yml')
37
+ config = YAML.load(File.read('VERSION.yml'))
38
+ version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
39
+ else
40
+ version = ""
41
+ end
42
+
43
+ rdoc.rdoc_dir = 'rdoc'
44
+ rdoc.title = "distributed_logreader #{version}"
45
+ rdoc.rdoc_files.include('README*')
46
+ rdoc.rdoc_files.include('lib/**/*.rb')
47
+ end
48
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.0
@@ -0,0 +1,80 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = %q{distributed_logreader}
5
+ s.version = "0.2.0"
6
+
7
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
8
+ s.authors = ["Gary Tsang"]
9
+ s.date = %q{2009-07-30}
10
+ s.email = %q{gary@garru.com}
11
+ s.extra_rdoc_files = [
12
+ "LICENSE",
13
+ "README.rdoc"
14
+ ]
15
+ s.files = [
16
+ ".gitignore",
17
+ "LICENSE",
18
+ "README.rdoc",
19
+ "Rakefile",
20
+ "VERSION",
21
+ "distributed_logreader.gemspec",
22
+ "lib/distributed_logreader.rb",
23
+ "lib/distributed_logreader/achiver.rb",
24
+ "lib/distributed_logreader/archiver/date_dir.rb",
25
+ "lib/distributed_logreader/distributed_log_reader.rb",
26
+ "lib/distributed_logreader/distributed_log_reader/rotater_reader.rb",
27
+ "lib/distributed_logreader/distributer.rb",
28
+ "lib/distributed_logreader/distributer/simple_thread_pool.rb",
29
+ "lib/distributed_logreader/log_reader.rb",
30
+ "lib/distributed_logreader/selector.rb",
31
+ "lib/distributed_logreader/selector/rotating_log.rb",
32
+ "lib/distributed_logreader/util.rb",
33
+ "spec/archiver/date_dir_spec.rb",
34
+ "spec/archiver_spec.rb",
35
+ "spec/distributed_log_reader/rotater_reader_spec.rb",
36
+ "spec/distributed_log_reader_spec.rb",
37
+ "spec/distributer/simple_thread_pool_spec.rb",
38
+ "spec/distributer_spec.rb",
39
+ "spec/fixtures/copytruncate/test",
40
+ "spec/fixtures/copytruncate/test.1",
41
+ "spec/fixtures/copytruncate/test_current",
42
+ "spec/fixtures/logrotate/test-20090101",
43
+ "spec/fixtures/logrotate/test-20090102",
44
+ "spec/fixtures/symlink/test",
45
+ "spec/fixtures/symlink/test_older_sym",
46
+ "spec/fixtures/test_file",
47
+ "spec/log_reader_spec.rb",
48
+ "spec/selector/rotating_log_spec.rb",
49
+ "spec/selector_spec.rb",
50
+ "spec/spec_helper.rb"
51
+ ]
52
+ s.has_rdoc = true
53
+ s.homepage = %q{http://github.com/garru/distributed_logreader}
54
+ s.rdoc_options = ["--charset=UTF-8"]
55
+ s.require_paths = ["lib"]
56
+ s.rubygems_version = %q{1.3.1}
57
+ s.summary = %q{TODO}
58
+ s.test_files = [
59
+ "spec/archiver/date_dir_spec.rb",
60
+ "spec/archiver_spec.rb",
61
+ "spec/distributed_log_reader/rotater_reader_spec.rb",
62
+ "spec/distributed_log_reader_spec.rb",
63
+ "spec/distributer/simple_thread_pool_spec.rb",
64
+ "spec/distributer_spec.rb",
65
+ "spec/log_reader_spec.rb",
66
+ "spec/selector/rotating_log_spec.rb",
67
+ "spec/selector_spec.rb",
68
+ "spec/spec_helper.rb"
69
+ ]
70
+
71
+ if s.respond_to? :specification_version then
72
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
73
+ s.specification_version = 2
74
+
75
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
76
+ else
77
+ end
78
+ else
79
+ end
80
+ end
@@ -0,0 +1,11 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
2
+ require 'distributed_logreader/selector.rb'
3
+ require 'distributed_logreader/achiver.rb'
4
+ require 'distributed_logreader/util.rb'
5
+ require 'distributed_logreader/distributed_log_reader'
6
+ require 'distributed_logreader/distributed_log_reader/rotater_reader'
7
+ require 'logger'
8
+
9
+ $dlog_logger = Logger.new("distributed_logreader.log")
10
+ $dlog_logger.level = Logger::INFO
11
+ $dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
@@ -0,0 +1,10 @@
1
+ # This abstract class defines the interface that handles log file archiving
2
+
3
+ module DLogReader
4
+ class Archiver
5
+ # archive file as you see fit
6
+ def archive(filename)
7
+ raise NotImplementedError.new("archive not implemented. Are you sure you created a conrete class?")
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,33 @@
1
+ require 'fileutils'
2
+ module DLogReader
3
+ class DateDir < Archiver
4
+ include FileUtils
5
+ attr_accessor :base_backup_dir
6
+
7
+ def initialize(backup_dir)
8
+ self.base_backup_dir = backup_dir
9
+ end
10
+
11
+ def backup(file)
12
+ mv(file, backup_dir) unless base_backup_dir.nil?
13
+
14
+ end
15
+
16
+ protected
17
+
18
+ def backup_dir
19
+ time = Time.now
20
+ directory_structure = [base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s]
21
+ temp_dir = []
22
+ directory_structure.each do |x|
23
+ temp_dir << x
24
+ temp_file = File.join(temp_dir)
25
+ unless File.exist?(temp_file)
26
+ mkdir(temp_file)
27
+ end
28
+ end
29
+ File.join(temp_dir)
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,39 @@
1
+ require File.join(File.dirname(__FILE__), 'distributer', 'simple_thread_pool')
2
+ require File.join(File.dirname(__FILE__), 'selector', 'rotating_log')
3
+ require File.join(File.dirname(__FILE__), 'archiver', 'date_dir')
4
+ require File.join(File.dirname(__FILE__), 'log_reader')
5
+
6
+ module DLogReader
7
+ class DistributedLogReader
8
+ attr_accessor :distributer, :filename
9
+ attr_reader :log_reader
10
+ def initialize(filename, worker, num_threads = 10)
11
+ self.filename = filename
12
+ self.distributer = SimpleThreadPool.new(worker, num_threads)
13
+ end
14
+
15
+ # selector/archiver seem to be strongly connected. it's possible it
16
+ # needs to be moved into LogReader
17
+ def process
18
+ pre_process
19
+ @log_reader = LogReader.new(log_file) do |line|
20
+ self.distributer.process(line)
21
+ end
22
+ @log_reader.run
23
+ self.distributer.join
24
+ post_process
25
+ end
26
+
27
+ def log_file
28
+ self.filename
29
+ end
30
+
31
+ #predefined hooks
32
+ def pre_process
33
+ end
34
+
35
+ #predefined hooks
36
+ def post_process
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,21 @@
1
+ module DLogReader
2
+ class RotaterReader < DistributedLogReader
3
+ attr_accessor :selector, :archiver
4
+ attr_reader :log_reader
5
+ def initialize(filename, backupdir, worker, num_threads = 10)
6
+ super(filename, worker, num_threads)
7
+ self.selector = RotatingLog.new
8
+ self.archiver = DateDir.new(backupdir)
9
+ end
10
+
11
+ def log_file
12
+ @log_file ||= begin
13
+ selector.file_to_process(filename)
14
+ end
15
+ end
16
+
17
+ def post_process
18
+ self.archiver.backup(log_file)
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,13 @@
1
+ module DLogReader
2
+ class Distributer
3
+ attr_accessor :worker
4
+
5
+ def initialize(worker)
6
+ self.worker = worker
7
+ end
8
+
9
+ def process(line)
10
+ worker.call(line)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,36 @@
1
+ require 'thread'
2
+
3
+ module DLogReader
4
+ class SimpleThreadPool
5
+ attr_accessor :num_threads, :worker, :thread_pool, :queue
6
+
7
+ def initialize(worker, num_threads = 10)
8
+ self.worker = worker
9
+ self.num_threads = num_threads
10
+ self.queue = Queue.new
11
+ num_threads.times do
12
+ create_thread
13
+ end
14
+ end
15
+
16
+ def process(line)
17
+ self.queue << line
18
+ end
19
+
20
+ def join
21
+ while(queue.size > 0)
22
+ sleep 0.1
23
+ end
24
+ end
25
+
26
+ protected
27
+ def create_thread
28
+ Thread.new do
29
+ loop do
30
+ line = self.queue.pop
31
+ self.worker.call(line)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,56 @@
1
+ require 'digest/md5'
2
+ module DLogReader
3
+ class LogReader
4
+ attr_accessor :filename
5
+ attr_writer :statefile
6
+
7
+ def initialize(filename, &b)
8
+ self.filename = filename
9
+ @b = b
10
+ end
11
+
12
+ def run
13
+ raise IOError.new("no file given") if filename.nil?
14
+ raise IOError.new("File not readable") unless File.readable?(filename)
15
+ f = File.open(filename, "r+")
16
+ load_saved_state(f)
17
+ raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
18
+ f.each_line do |line|
19
+ @b.call(line)
20
+ end
21
+ save_state(f)
22
+ f.flock(File::LOCK_UN)
23
+ end
24
+
25
+ def statefile
26
+ @statefile ||= begin
27
+ log_basename = File.basename(filename)
28
+ File.join("/tmp", "log_state_#{log_basename}")
29
+ end
30
+ end
31
+
32
+ protected
33
+
34
+ def load_saved_state(log_filehandle)
35
+ return unless File.exists?(statefile) && !(state = File.read(statefile)).nil?
36
+ pos, l_digest = Marshal.load(state)
37
+ return if File.size(log_filehandle) < pos
38
+ log_filehandle.pos = pos if digest(log_filehandle, pos) == l_digest
39
+ end
40
+
41
+ def save_state(log_filehandle)
42
+ File.open(statefile, "w") do |f|
43
+ f.write(Marshal.dump([log_filehandle.pos, digest(log_filehandle, log_filehandle.pos)]))
44
+ end
45
+ end
46
+
47
+ def digest(log_filehandle, position)
48
+ log_filehandle.pos = 0
49
+ read_length = [position, 50].min
50
+ l = log_filehandle.read(read_length)
51
+ f_digest = Digest::MD5.hexdigest(l)
52
+ log_filehandle.pos = position
53
+ f_digest
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,12 @@
1
+ module DLogReader
2
+ # This abstract class defines the interface to decide which log
3
+ # file to read. The identity strategy is the simplist, to return the file
4
+ # inputed. However, to handle rotating log files, we'll need some more complex
5
+ # strategies.
6
+ class Selector
7
+ # determines the file to process from file path input
8
+ def file_to_process(file_or_dir)
9
+ raise NotImplementedError.new, "file_to_process not implemented. Are you sure you created a conrete class?"
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,31 @@
1
+ module DLogReader
2
+ # This class chooses the oldest log file in the directory that matches the
3
+ # input filename. This should work with a variety of log rotating schemes:
4
+ # including copytruncate and date suffix.
5
+ class RotatingLog < Selector
6
+
7
+ def file_to_process(file_or_dir)
8
+ if File.directory?(file_or_dir)
9
+ directory = file_or_dir
10
+ basename = '/'
11
+ else
12
+ directory = File.dirname(file_or_dir)
13
+ basename = File.basename(file_or_dir)
14
+ end
15
+ oldest_logfile(directory, basename)
16
+ end
17
+
18
+ protected
19
+
20
+ def oldest_logfile(directory, basename)
21
+ file_list = Dir[File.join(directory, "#{basename}*")]
22
+ file_list.reject!{|x| symlink_file_in_dir?(x)}
23
+ file = file_list.size > 0 ? file_list.sort_by{|a| File.new(a).mtime}.first : nil
24
+ end
25
+
26
+ # returns true if filename is a symlink and its referring to a file already inside the current directory
27
+ def symlink_file_in_dir?(filename)
28
+ File.symlink?(filename) && File.dirname(File.readlink(filename)) == File.dirname(filename)
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,7 @@
1
+ module DLogReader
2
+ module Util
3
+ def logger
4
+ $dlog_logger
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,25 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+ require 'distributed_logreader/archiver/date_dir'
3
+ require 'fileutils'
4
+
5
+ describe "DLogReader::DateDir" do
6
+ before(:all) do
7
+ FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
8
+ @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
9
+ @base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
10
+ time = Time.now
11
+ @backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
12
+ @archiver = DLogReader::DateDir.new(@base_backup_dir)
13
+ end
14
+
15
+ describe "backup" do
16
+ it "should move file into Y/M/D backup directory" do
17
+ @archiver.backup(@file_path)
18
+ File.exist?(@backup_dir).should == true
19
+ end
20
+ end
21
+
22
+ after(:all) do
23
+ FileUtils.rm_r(@base_backup_dir)
24
+ end
25
+ end
@@ -0,0 +1,13 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "DLogReader::LogArchiver" do
4
+ before(:all) do
5
+ @archiver = DLogReader::Archiver.new
6
+ end
7
+
8
+ describe "archive" do
9
+ it "should raise NotImplementedError" do
10
+ lambda{ @archiver.archive('dummy_file') }.should raise_error(NotImplementedError)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,26 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+ require 'fileutils'
3
+
4
+ describe "DLogReader::RotaterLogreader" do
5
+ before(:all) do
6
+ FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
7
+ @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
8
+ @base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
9
+ time = Time.now
10
+ @backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
11
+ @logreader = DLogReader::RotaterReader.new(@file_path, @base_backup_dir, lambda{|x| puts x})
12
+ end
13
+
14
+ describe "process" do
15
+ it 'should' do
16
+ @logreader.process
17
+ File.exist?(@backup_dir).should == true
18
+ File.exist?(@file_path).should == false
19
+ end
20
+ end
21
+
22
+ after(:all) do
23
+ FileUtils.rm_r(@base_backup_dir)
24
+ FileUtils.rm_r(@logreader.log_reader.statefile)
25
+ end
26
+ end
@@ -0,0 +1,20 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+ require 'fileutils'
3
+
4
+ describe "DLogReader::DistributedLogreader" do
5
+ before(:all) do
6
+ FileUtils.cp(File.join(File.dirname(__FILE__), 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), 'fixtures', 'test_file2'))
7
+ @file_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_file2')
8
+ @logreader = DLogReader::DistributedLogReader.new(@file_path, lambda{|x| puts x})
9
+ end
10
+
11
+ describe "process" do
12
+ it 'should' do
13
+ @logreader.process
14
+ end
15
+ end
16
+
17
+ after(:all) do
18
+ FileUtils.rm_r(@logreader.log_reader.statefile)
19
+ end
20
+ end
@@ -0,0 +1,17 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+ require 'distributed_logreader/distributer/simple_thread_pool'
3
+
4
+ describe "DLogReader::SimpleThreadPool" do
5
+ before(:all) do
6
+ @thread_pool = DLogReader::SimpleThreadPool.new(lambda{|x| x}, 10)
7
+ end
8
+
9
+ describe "process" do
10
+ it 'should process with lots of threads' do
11
+ 100.times do |x|
12
+ @thread_pool.process(x.to_s)
13
+ end
14
+ @thread_pool.join
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,13 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+ require 'distributed_logreader/distributer'
3
+
4
+ describe "DLogReader::Distributer" do
5
+ before(:all) do
6
+ @distributer = DLogReader::Distributer.new(lambda{|x| x})
7
+ end
8
+
9
+ describe "process" do
10
+ it "should raise NotImplementedError" do
11
+ end
12
+ end
13
+ end
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
@@ -0,0 +1,4 @@
1
+ This is a test log file.
2
+ With many lines.
3
+ Lots and Lots of lines.
4
+ Just kidding this is the last line.
@@ -0,0 +1,57 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+ require 'distributed_logreader/log_reader'
3
+ require 'fileutils'
4
+
5
+ describe "DLogReader::LogReader" do
6
+ before(:all) do
7
+ test_file = File.join(File.dirname(__FILE__), 'fixtures', 'test_file')
8
+ FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading'))
9
+ @test_cp = File.join(File.dirname(__FILE__), 'fixtures', 'logreading', 'test')
10
+ FileUtils.cp(test_file, @test_cp)
11
+ test_fh = File.open(test_file)
12
+
13
+ @reader = DLogReader::LogReader.new(@test_cp) do |line|
14
+ unless test_fh.readline == line
15
+ raise RuntimeError.new, 'you messed up bud'
16
+ end
17
+ end
18
+ @test_line = "this is an added line. this should be read first\n"
19
+ @state_writer = DLogReader::LogReader.new(@test_cp){|line| line;}
20
+ @state_reader = DLogReader::LogReader.new(@test_cp) do |line|
21
+ unless line == @test_line
22
+ raise RuntimeError.new, 'you messed up worse'
23
+ end
24
+ end
25
+ end
26
+
27
+ describe "run" do
28
+ it 'should read log files' do
29
+ lambda{@reader.run}.should_not raise_error
30
+ end
31
+
32
+ it 'should resume from last access' do
33
+ #lets read to the end of file and write state
34
+ lambda{@state_writer.run}.should_not raise_error
35
+ fh = File.open(@test_cp, 'a')
36
+ fh.write(@test_line)
37
+ fh.close
38
+ lambda{@state_reader.run}.should_not raise_error
39
+ end
40
+
41
+ it 'should detect if log is different from last and to start from beg of file' do
42
+ lambda{@state_writer.run}.should_not raise_error
43
+ fh = File.open(@test_cp, 'w')
44
+ fh.write(@test_line)
45
+ fh.close
46
+ lambda{@state_reader.run}.should_not raise_error
47
+ fh = File.open(@test_cp, 'w')
48
+ fh.write('')
49
+ fh.close
50
+ lambda{@state_reader.run}.should_not raise_error
51
+ end
52
+ end
53
+
54
+ after(:all) do
55
+ FileUtils.rm_r(File.dirname(@test_cp))
56
+ end
57
+ end
@@ -0,0 +1,24 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+ require 'distributed_logreader/selector/rotating_log'
3
+ require 'fileutils'
4
+
5
+ describe "DLogReader::RotatingLog" do
6
+ before(:all) do
7
+ @chooser = DLogReader::RotatingLog.new
8
+ end
9
+
10
+ describe "file_to_process" do
11
+ it "should pick the oldest file for logs in copytruncate format (file)" do
12
+ @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test.1')
13
+ end
14
+
15
+ it "should pick the oldest file in timestamp suffix log format (dirname)" do
16
+ @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate', 'test-20090101')
17
+ end
18
+
19
+ it "should pick the oldest file ignoring symlinks pointing to files already in dir" do
20
+ @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink', 'test')
21
+ end
22
+ end
23
+
24
+ end
@@ -0,0 +1,13 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "DLogReader::Selector" do
4
+ before(:all) do
5
+ @chooser = DLogReader::Selector.new
6
+ end
7
+
8
+ describe "file_to_process" do
9
+ it "should raise NotImplementedError" do
10
+ lambda{ @chooser.file_to_process('dummy_file') }.should raise_error(NotImplementedError)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,9 @@
1
+ require 'spec'
2
+
3
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
4
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
5
+ require 'distributed_logreader'
6
+
7
+ Spec::Runner.configure do |config|
8
+
9
+ end
metadata ADDED
@@ -0,0 +1,98 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: garru-distributed_logreader
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Gary Tsang
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-07-30 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description:
17
+ email: gary@garru.com
18
+ executables: []
19
+
20
+ extensions: []
21
+
22
+ extra_rdoc_files:
23
+ - LICENSE
24
+ - README.rdoc
25
+ files:
26
+ - .gitignore
27
+ - LICENSE
28
+ - README.rdoc
29
+ - Rakefile
30
+ - VERSION
31
+ - distributed_logreader.gemspec
32
+ - lib/distributed_logreader.rb
33
+ - lib/distributed_logreader/achiver.rb
34
+ - lib/distributed_logreader/archiver/date_dir.rb
35
+ - lib/distributed_logreader/distributed_log_reader.rb
36
+ - lib/distributed_logreader/distributed_log_reader/rotater_reader.rb
37
+ - lib/distributed_logreader/distributer.rb
38
+ - lib/distributed_logreader/distributer/simple_thread_pool.rb
39
+ - lib/distributed_logreader/log_reader.rb
40
+ - lib/distributed_logreader/selector.rb
41
+ - lib/distributed_logreader/selector/rotating_log.rb
42
+ - lib/distributed_logreader/util.rb
43
+ - spec/archiver/date_dir_spec.rb
44
+ - spec/archiver_spec.rb
45
+ - spec/distributed_log_reader/rotater_reader_spec.rb
46
+ - spec/distributed_log_reader_spec.rb
47
+ - spec/distributer/simple_thread_pool_spec.rb
48
+ - spec/distributer_spec.rb
49
+ - spec/fixtures/copytruncate/test
50
+ - spec/fixtures/copytruncate/test.1
51
+ - spec/fixtures/copytruncate/test_current
52
+ - spec/fixtures/logrotate/test-20090101
53
+ - spec/fixtures/logrotate/test-20090102
54
+ - spec/fixtures/symlink/test
55
+ - spec/fixtures/symlink/test_older_sym
56
+ - spec/fixtures/test_file
57
+ - spec/log_reader_spec.rb
58
+ - spec/selector/rotating_log_spec.rb
59
+ - spec/selector_spec.rb
60
+ - spec/spec_helper.rb
61
+ has_rdoc: true
62
+ homepage: http://github.com/garru/distributed_logreader
63
+ licenses:
64
+ post_install_message:
65
+ rdoc_options:
66
+ - --charset=UTF-8
67
+ require_paths:
68
+ - lib
69
+ required_ruby_version: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ version: "0"
74
+ version:
75
+ required_rubygems_version: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - ">="
78
+ - !ruby/object:Gem::Version
79
+ version: "0"
80
+ version:
81
+ requirements: []
82
+
83
+ rubyforge_project:
84
+ rubygems_version: 1.3.5
85
+ signing_key:
86
+ specification_version: 2
87
+ summary: TODO
88
+ test_files:
89
+ - spec/archiver/date_dir_spec.rb
90
+ - spec/archiver_spec.rb
91
+ - spec/distributed_log_reader/rotater_reader_spec.rb
92
+ - spec/distributed_log_reader_spec.rb
93
+ - spec/distributer/simple_thread_pool_spec.rb
94
+ - spec/distributer_spec.rb
95
+ - spec/log_reader_spec.rb
96
+ - spec/selector/rotating_log_spec.rb
97
+ - spec/selector_spec.rb
98
+ - spec/spec_helper.rb