ralf 0.1.5 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/ralf.rb ADDED
@@ -0,0 +1,173 @@
1
+ require 'rubygems'
2
+ require 'right_aws'
3
+ require 'logmerge'
4
+ require 'ftools'
5
+ require 'ralf/config'
6
+ require 'ralf/bucket'
7
+ require 'chronic'
8
+
9
+ class Ralf
10
+
11
+ private
12
+ RLIMIT_NOFILE_HEADROOM = 100 # number of file descriptors to allocate above number of logfiles
13
+
14
+ public
15
+
16
+ ROOT_DEFAULT_CONFIG_FILE = '/etc/ralf.conf'
17
+ USER_DEFAULT_CONFIG_FILE = '~/.ralf.conf'
18
+
19
+ AMAZON_LOG_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
20
+
21
+ # The current configuration.
22
+ attr_reader :config
23
+
24
+ # Instance of RightAws::S3 used by Ralf to talk to Amazon S3.
25
+ attr_reader :s3
26
+
27
+ # Create instance and run with the specified parameters
28
+ def self.run(options)
29
+ list = options.delete(:list)
30
+
31
+ ralf = Ralf.new(options)
32
+ if list
33
+ ralf.list
34
+ else
35
+ ralf.run
36
+ end
37
+ end
38
+
39
+ # Create new Ralf instance
40
+ def initialize(options = {})
41
+ initial_options = options.dup
42
+
43
+ @config = load_config(initial_options.delete(:config_file))
44
+
45
+ config.merge!(initial_options)
46
+ config.validate!
47
+
48
+ RightAws::RightAwsBaseInterface.caching = true # enable caching to speed up
49
+ Bucket.s3 = RightAws::S3.new(config.aws_access_key_id, config.aws_secret_access_key,
50
+ :protocol => 'http', :port => 80,
51
+ :logger => Logger.new('aws' == config.debug? ? $stdout : StringIO.new)
52
+ )
53
+ end
54
+
55
+ # For all buckets for all dates in the configured range download the available
56
+ # log files. After downloading, merge the log files and convert the format
57
+ # from Amazon Log Format to Apache Common Log Format.
58
+ def run(output_file = nil)
59
+ config.output_file = output_file unless output_file.nil?
60
+
61
+ raise ArgumentError.new("--output-file required") if config.output_file_missing?
62
+ raise ArgumentError.new("--output-file requires ':bucket' variable") if (config.buckets.nil? || config.buckets.size > 1) and !(config.output_file_format =~ /:bucket/)
63
+
64
+ puts "Processing range #{config.range}" if config.debug?
65
+
66
+ # iterate over all buckets
67
+ Bucket.each(config.buckets) do |bucket|
68
+
69
+ print "#{bucket.name}: " if config.debug?
70
+
71
+ # iterate over the full range
72
+ log_files = []
73
+ config.range.each do |date|
74
+ dir = config.cache_dir(:bucket => bucket.name, :date => date)
75
+ log_files << Ralf.download_logs(bucket, date, dir)
76
+ end
77
+ log_files.flatten!
78
+
79
+ # determine output file names
80
+ output_log = config.output_file(:date => config.range.end, :bucket => bucket.name)
81
+ merged_log = output_log + ".alf"
82
+
83
+ # create directory for output file
84
+ File.makedirs(File.dirname(merged_log))
85
+
86
+ # merge the log files
87
+ Ralf.merge(merged_log, log_files)
88
+
89
+ # convert to common log format
90
+ Ralf.convert_to_common_log_format(merged_log, output_log)
91
+
92
+ puts "#{log_files.size} files" if config.debug?
93
+ end
94
+ end
95
+
96
+ # List all buckets with the logging info.
97
+ def list(with_logging = false)
98
+ puts "Listing buckets..." if config.debug?
99
+
100
+ Bucket.each(config.buckets, with_logging) do |bucket|
101
+ print "#{bucket.name}"
102
+ puts bucket.logging_enabled? ? " [#{bucket.targetbucket}/#{bucket.targetprefix}]" : " [-]"
103
+ end
104
+
105
+ nil
106
+ end
107
+
108
+ private
109
+
110
+ # Download log files for +bucket+ and +date+ to +dir+.
111
+ def self.download_logs(bucket, date, dir)
112
+ File.makedirs(dir)
113
+
114
+ # iterate over the available log files, saving them to disk and
115
+ log_files = []
116
+ bucket.each_log(date) do |log|
117
+ log_files << log.save_to_dir(dir)
118
+ end
119
+ log_files
120
+ end
121
+
122
+ # Takes an array of log file names and merges them on ascending timestamp
123
+ # into +output_file+ name. Assumes the +log_files+ are sorted
124
+ # on ascending timestamp.
125
+ def self.merge(output_file, log_files)
126
+ update_rlimit_nofile(log_files.size)
127
+ File.open(output_file, 'w') do |out_file|
128
+ LogMerge::Merger.merge out_file, *log_files
129
+ end
130
+ end
131
+
132
+ # Convert the input_log file to Apache Common Log Format into output_log
133
+ def self.convert_to_common_log_format(input_log, output_log)
134
+ out_file = File.open(output_log, 'w')
135
+ File.open(input_log, 'r') do |in_file|
136
+ while (line = in_file.gets)
137
+ out_file.puts(translate_to_clf(line))
138
+ end
139
+ end
140
+ out_file.close
141
+ end
142
+
143
+ def self.translate_to_clf(line)
144
+ if line =~ AMAZON_LOG_FORMAT
145
+ # host, date, ip, acl, request, status, bytes, agent = $2, $3, $4, $5, $9, $10, $12, $17
146
+ "%s - %s [%s] \"%s\" %d %s \"%s\" \"%s\"" % [$4, $5, $3, $9, $10, $12, $16, $17]
147
+ else
148
+ $stderr.puts "# ERROR: #{line}"
149
+ end
150
+ end
151
+
152
+ def load_config(cli_config_file)
153
+ result = nil
154
+ if cli_config_file
155
+ result = Ralf::Config.load_file(cli_config_file) unless cli_config_file.empty?
156
+ else
157
+ config_file = (0 == Process.uid ? ROOT_DEFAULT_CONFIG_FILE : File.expand_path(USER_DEFAULT_CONFIG_FILE))
158
+ result = Ralf::Config.load_file(config_file) if File.exist?(config_file)
159
+ end
160
+ result || Ralf::Config.new
161
+ end
162
+
163
+ def self.update_rlimit_nofile(number_of_files)
164
+ new_rlimit_nofile = number_of_files + RLIMIT_NOFILE_HEADROOM
165
+
166
+ # getrlimit returns array with soft and hard limit [soft, hard]
167
+ rlimit_nofile = Process::getrlimit(Process::RLIMIT_NOFILE)
168
+ if new_rlimit_nofile > rlimit_nofile.first
169
+ Process.setrlimit(Process::RLIMIT_NOFILE, new_rlimit_nofile) rescue nil
170
+ end
171
+ end
172
+
173
+ end
@@ -0,0 +1,7 @@
1
+ 10.34.211.35 - F7CECCD7A21B4B96 [10/Feb/2010:07:16:19 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 456 - "-" "Jakarta Commons-HttpClient/3.0" -
2
+ 10.34.211.35 - E415BCEA05AFB84F [10/Feb/2010:07:16:19 +0000] "POST /soap/ HTTP/1.1" 200 - 797 1003 74 41 "-" "Axis/1.3" -
3
+ 10.36.142.18 - BE89D3B6953D719E [10/Feb/2010:07:16:28 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 433 - "-" "Jakarta Commons-HttpClient/3.0" -
4
+ 10.36.142.18 - E15F489437E852EA [10/Feb/2010:07:16:28 +0000] "POST /soap/ HTTP/1.1" 200 - 797 954 36 14 "-" "Axis/1.3" -
5
+ 10.32.219.38 - 784FD457838EFF42 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -
6
+ 10.32.219.38 - 6E239BC5A4AC757C [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -
7
+ 10.217.37.15 - 0B76C90B3634290B [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -
@@ -0,0 +1,48 @@
1
+ ---
2
+ -
3
+ :name: 'test1'
4
+ :logging_info:
5
+ :enabled: true
6
+ :targetbucket: test1
7
+ :targetprefix: logs/
8
+ :keys:
9
+ -
10
+ :name: 'logs/2010-02-10-00-05-32-ZDRFGTCKUYVJCT'
11
+ -
12
+ :name: 'logs/2010-02-11-00-05-32-ZDRFGTCKUYVJCT'
13
+ -
14
+ :name: 'test2'
15
+ :logging_info:
16
+ :enabled: true
17
+ :targetbucket: target_test2
18
+ :targetprefix: logs/access_log-
19
+ :keys:
20
+ -
21
+ :name: 'logs/access_log-2010-02-12-00-05-32-ZDRFGTCKUYVJCT'
22
+ -
23
+ :name: 'logs/access_log-2010-02-13-00-05-32-ZDRFGTCKUYVJCT'
24
+ -
25
+ :name: 'test3'
26
+ :logging_info:
27
+ :enabled: false
28
+ :targetbucket: ''
29
+ :targetprefix: ''
30
+ :keys: []
31
+ -
32
+ :name: 'test4'
33
+ :logging_info:
34
+ :enabled: true
35
+ :targetbucket: test4
36
+ :targetprefix: logs/
37
+ :keys:
38
+ -
39
+ :name: 'logs/2010-02-14-00-05-32-ZDRFGTCKUYVJCT'
40
+ -
41
+ :name: 'logs/2010-02-15-00-05-32-ZDRFGTCKUYVJCT'
42
+ -
43
+ :name: 'target_test2'
44
+ :logging_info:
45
+ :enabled: false
46
+ :targetbucket: ''
47
+ :targetprefix: ''
48
+ :keys: []
@@ -0,0 +1,119 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ require 'ralf'
4
+ require 'ralf/bucket'
5
+
6
+ describe Ralf::Bucket do
7
+
8
+ before(:all) do
9
+ @valid_logging_info = {
10
+ :enabled => true,
11
+ :targetbucket => 'targetbucket',
12
+ :targetprefix => 'logs/',
13
+ }
14
+ end
15
+
16
+ it "should initialize properly" do
17
+ mock_S3
18
+
19
+ name = 's3_bucket'
20
+ logging_info = {
21
+ :enabled => true,
22
+ :targetbucket => 's3_bucket',
23
+ :targetprefix => 'logs/',
24
+ }
25
+ bucket = mock(name)
26
+ bucket.should_receive(:logging_info).and_return(logging_info)
27
+ bucket.should_receive(:name).and_return(name)
28
+
29
+ lambda {
30
+ Ralf::Bucket.new(bucket)
31
+ }.should_not raise_error
32
+ end
33
+
34
+ it "should set targetbucket to bucket returned by logging_info" do
35
+ mock_S3
36
+
37
+ name = 's3_bucket'
38
+ targetbucket_name = 's3_targetbucket'
39
+ logging_info = {
40
+ :enabled => true,
41
+ :targetbucket => targetbucket_name,
42
+ :targetprefix => 'logs/',
43
+ }
44
+ bucket = mock(name)
45
+ bucket.should_receive(:logging_info).and_return(logging_info)
46
+ bucket.should_receive(:name).and_return(name)
47
+
48
+ Ralf::Bucket.new(bucket)
49
+ end
50
+
51
+ it "should support iteration over all buckets" do
52
+ mock_S3
53
+
54
+ yielded_buckets = []
55
+ Ralf::Bucket.each do |bucket|
56
+ yielded_buckets << bucket
57
+ end
58
+ yielded_buckets.should have(@enabled_buckets_count).items
59
+ yielded_buckets.each { |bucket| bucket.name.should_not be_nil }
60
+ end
61
+
62
+ it "should support iteration over specific buckets" do
63
+ mock_S3
64
+
65
+ yielded_buckets = []
66
+ Ralf::Bucket.each(@example_buckets.keys) do |bucket|
67
+ yielded_buckets << bucket
68
+ end
69
+ yielded_buckets.should have(@enabled_buckets_count).items
70
+ yielded_buckets.each do |bucket|
71
+ bucket.logging_enabled?.should eql(@example_buckets[bucket.name].logging_info[:enabled])
72
+ bucket.targetbucket.should eql(@example_buckets[bucket.name].logging_info[:targetbucket])
73
+ bucket.targetprefix.should eql(@example_buckets[bucket.name].logging_info[:targetprefix])
74
+ end
75
+ end
76
+
77
+ it "should support iterating over all logs in a bucket" do
78
+ mock_S3
79
+
80
+ bucket_mock = @example_buckets['test1']
81
+ key1, key2 = mock('key1'), mock('key2')
82
+ keys = [key1, key2]
83
+ bucket_mock.should_receive(:keys).with(:prefix => 'logs/2010-05-17').and_return(keys)
84
+
85
+ expected_logs = []
86
+ keys.each do |key|
87
+ log = mock('Log')
88
+ Ralf::Log.should_receive(:new).with(key, @example_buckets['test1'].logging_info[:targetprefix]).and_return(log)
89
+ expected_logs << log
90
+ end
91
+
92
+ bucket = Ralf::Bucket.new(bucket_mock)
93
+
94
+ yielded_logs = []
95
+ bucket.each_log(Date.new(2010,05,17)) do |log|
96
+ yielded_logs << log
97
+ end
98
+ yielded_logs.should have(keys.size).items
99
+ yielded_logs.should eql(expected_logs)
100
+ end
101
+
102
+ def mock_S3
103
+ @s3_mock = mock('S3')
104
+ Ralf::Bucket.s3 = @s3_mock
105
+
106
+ # load example buckets (2 disabled)
107
+ @example_buckets = load_example_bucket_mocks
108
+ @enabled_buckets_count = @example_buckets.size - 2
109
+
110
+ # make s3_mock return individual buckets
111
+ @s3_mock.should_receive(:bucket).any_number_of_times do |name|
112
+ @example_buckets[name]
113
+ end
114
+
115
+ # make s3_mock return all buckets
116
+ @s3_mock.should_receive(:buckets).any_number_of_times.and_return(@example_buckets.values)
117
+ end
118
+
119
+ end
@@ -0,0 +1,131 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ require 'ralf'
4
+ require 'ralf/config'
5
+
6
+ describe Ralf::Config do
7
+
8
+ before(:all) do
9
+ ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = nil, nil
10
+
11
+ @aws_credentials = {
12
+ :aws_access_key_id => 'access_key_id',
13
+ :aws_secret_access_key => 'secret_key',
14
+ }
15
+
16
+ @valid_options = {
17
+ :output_file => 'my_log.log',
18
+ }.merge(@aws_credentials)
19
+
20
+ @date = Date.strptime('2010-02-09')
21
+ @bucket = 'my.bucket.org'
22
+ end
23
+
24
+ it "should initialize properly" do
25
+ config = Ralf::Config.new
26
+ end
27
+
28
+ it "should raise error for missing credentials" do
29
+ lambda {
30
+ config = Ralf::Config.new
31
+ config.validate!
32
+ }.should raise_error(Ralf::Config::ConfigurationError, 'aws_access_key_id missing, aws_secret_access_key missing')
33
+ end
34
+
35
+ it "should handle range assignment" do
36
+ now = Time.now
37
+ yesterday = Date.today - 1
38
+ Time.should_receive(:now).any_number_of_times.and_return(now)
39
+ config = Ralf::Config.new(@valid_options)
40
+ config.range = 'yesterday'
41
+ config.range.should eql(Range.new(yesterday, yesterday))
42
+ end
43
+
44
+ it "should interpolate date (:year, :month, :day) variables in output_file" do
45
+ config = Ralf::Config.new(@valid_options.merge(:output_file => ':year/:month/access.log'))
46
+ config.output_file(:date => @date).should eql('2010/02/access.log')
47
+ end
48
+
49
+ it "should interpolate :bucket variable in output_file" do
50
+ config = Ralf::Config.new(@valid_options.merge(:output_file => ':bucket.log'))
51
+ config.output_file(:date => @date, :bucket => @bucket).should eql('my.bucket.org.log')
52
+ end
53
+
54
+ it "should interpolate variables in cache_dir" do
55
+ config = Ralf::Config.new(@valid_options.merge(:cache_dir => ':year/:month/:bucket'))
56
+ config.cache_dir(:date => @date, :bucket => @bucket).should eql('2010/02/my.bucket.org')
57
+ end
58
+
59
+ it "should allow 'this month' with base 'yesterday'" do
60
+ Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
61
+ config = Ralf::Config.new(:range => 'this month', :now => 'yesterday')
62
+ config.range.to_s.should eql('2010-04-01..2010-04-30')
63
+ end
64
+
65
+ it "should merge options" do
66
+ Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
67
+ config = Ralf::Config.new
68
+ config.merge!(:range => 'this month', :now => 'yesterday')
69
+ config.range.to_s.should eql('2010-04-01..2010-04-30')
70
+ end
71
+
72
+ it "should support setting now after range and recompute range" do
73
+ Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
74
+ config = Ralf::Config.new
75
+ config.merge!(:range => 'this month')
76
+ config.merge!(:now => 'yesterday')
77
+ config.range.to_s.should eql('2010-04-01..2010-04-30')
78
+ end
79
+
80
+ it "should support setting range first then change now (1st day of month)" do
81
+ Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
82
+ config = Ralf::Config.new
83
+ config.merge!(:range => 'this month')
84
+ config.range.to_s.should eql('2010-05-01..2010-05-01')
85
+ config.merge!(:now => 'yesterday')
86
+ config.range.to_s.should eql('2010-04-01..2010-04-30')
87
+ end
88
+
89
+ it "should support setting range first then change now" do
90
+ Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 08 16:31:00 +0100 2010'))
91
+ config = Ralf::Config.new
92
+ config.merge!(:range => 'this month')
93
+ config.range.to_s.should eql('2010-05-01..2010-05-07')
94
+ config.merge!(:now => '2010-05-06')
95
+ config.range.to_s.should eql('2010-05-01..2010-05-06')
96
+ end
97
+
98
+ it "should set default cache_dir to '/var/log/ralf/:bucket' when running as root" do
99
+ Process.should_receive(:uid).and_return(0)
100
+ config = Ralf::Config.new
101
+ config.cache_dir_format.should eql('/var/log/ralf/:bucket')
102
+ end
103
+
104
+ it "should set default cache_dir to '~/.ralf/:bucket' when running as regular user" do
105
+ Process.should_receive(:uid).and_return(1)
106
+ File.should_receive(:expand_path).with('~/.ralf/:bucket').and_return('/Users/me/.ralf/:bucket')
107
+ config = Ralf::Config.new
108
+ config.cache_dir_format.should eql('/Users/me/.ralf/:bucket')
109
+ end
110
+
111
+ it "should allow overriding bucket with merge!" do
112
+ config = Ralf::Config.new(:buckets => ['bucket1', 'bucket2'])
113
+ config.buckets = ['bucket3']
114
+ config.buckets.should eql(['bucket3'])
115
+ end
116
+
117
+ it "should load configuration from file" do
118
+ YAML.should_receive(:load_file).with('my_config.yaml').and_return(@valid_options)
119
+ config = Ralf::Config.load_file('my_config.yaml')
120
+ config.output_file_format.should eql(@valid_options[:output_file])
121
+ end
122
+
123
+ it "should warn and continue (not raise exception) when unknown configuration variables are set" do
124
+ $stdout = StringIO.new
125
+ lambda {
126
+ config = Ralf::Config.new(:unknown_variable => 100)
127
+ }.should_not raise_error
128
+ $stdout.string.should eql("Warning: invalid configuration variable: unknown_variable\n")
129
+ end
130
+
131
+ end
@@ -0,0 +1,45 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ require 'ralf/interpolation'
4
+
5
+ describe Ralf::Interpolation do
6
+ before(:all) do
7
+ @bucket = 'bucket.mybuckets.org'
8
+ @date = Date.strptime('2010-02-09')
9
+ end
10
+
11
+ [
12
+ [':year', '2010'],
13
+ [':month', '02'],
14
+ [':day', '09'],
15
+ [':week', '06']
16
+ ].each do |match, result, bucket|
17
+ it "should replace '#{match}' with '#{result}'" do
18
+ Ralf::Interpolation.interpolate(match, :date => @date).should eql(result)
19
+ end
20
+ end
21
+
22
+ it "should raise an error when not all symbols could be interpolated" do
23
+ lambda {
24
+ Ralf::Interpolation.interpolate(':unknown', :date => @date)
25
+ }.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
26
+ end
27
+
28
+ it "should interpolate :bucket" do
29
+ Ralf::Interpolation.interpolate(':year/:month/:day/:bucket.log',
30
+ :date => @date, :bucket => @bucket).should eql('2010/02/09/bucket.mybuckets.org.log')
31
+ end
32
+
33
+ it "should require :bucket interpolation if bucket specified" do
34
+ lambda {
35
+ Ralf::Interpolation.interpolate(':year/:month/:day.log', {:date => @date, :bucket => @bucket}, [:bucket])
36
+ }.should raise_error(Ralf::Interpolation::VariableMissing, ':bucket variable missing')
37
+ end
38
+
39
+ it "should raise an error when :bucket can not be interpolated" do
40
+ lambda {
41
+ Ralf::Interpolation.interpolate(':bucket', :date => @date)
42
+ }.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
43
+ end
44
+
45
+ end
@@ -0,0 +1,63 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ require 'ralf'
4
+ require 'ralf/log'
5
+
6
+ describe Ralf::Log do
7
+
8
+ it "should initialize properly" do
9
+ key = mock('key')
10
+ prefix = mock('prefix')
11
+ lambda {
12
+ Ralf::Log.new(key, prefix)
13
+ }.should_not raise_error
14
+ end
15
+
16
+ it "should remove prefix when returing name" do
17
+ key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
18
+ log = Ralf::Log.new(key, 'log/access_log-')
19
+ log.name.should eql('2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
20
+ end
21
+
22
+ it "should save specified file to dir" do
23
+ key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
24
+ log = Ralf::Log.new(key, 'log/access_log-')
25
+
26
+ dir = '/var/log/s3'
27
+ filename = File.join(dir, log.name)
28
+ fileio = mock('File')
29
+ key.should_receive(:data).and_return('testdata')
30
+ File.should_receive(:open).with(filename, 'w').and_yield(fileio)
31
+ File.should_receive(:exist?).with(filename).and_return(false)
32
+ fileio.should_receive(:write).with('testdata')
33
+ log.save_to_dir(dir)
34
+ end
35
+
36
+ it "should not save file if it exists and caching enabled (default)" do
37
+ key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
38
+ log = Ralf::Log.new(key, 'log/access_log-')
39
+
40
+ dir = '/var/log/s3'
41
+ filename = File.join(dir, log.name)
42
+ fileio = mock('File')
43
+ File.should_receive(:exist?).with(filename).and_return(true)
44
+ File.should_not_receive(:open).with(filename, 'w')
45
+ log.save_to_dir(dir)
46
+ end
47
+
48
+ it "should not check file if use_cache = false" do
49
+ key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
50
+ log = Ralf::Log.new(key, 'log/access_log-')
51
+
52
+ dir = '/var/log/s3'
53
+ filename = File.join(dir, log.name)
54
+ fileio = mock('File')
55
+
56
+ key.should_receive(:data).and_return('testdata')
57
+ File.should_receive(:open).with(filename, 'w').and_yield(fileio)
58
+ File.should_not_receive(:exist?).with(filename)
59
+ fileio.should_receive(:write).with('testdata')
60
+ log.save_to_dir(dir, false)
61
+ end
62
+
63
+ end
@@ -0,0 +1,97 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ require 'ralf'
4
+ require 'ralf/option_parser'
5
+
6
+ describe Ralf::OptionParser do
7
+
8
+ before(:all) do
9
+ @valid_arguments = [
10
+ [ :buckets, '--buckets', [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
11
+ [ :buckets, '-b', [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
12
+
13
+ [ :range, '--range', ['today'] ],
14
+ [ :range, '-r', ['today'] ],
15
+
16
+ [ :now, '--now', 'yesterday' ],
17
+ [ :now, '-t', 'yesterday' ],
18
+
19
+ [ :output_file, '--output-file', '/var/log/s3/:year/:month/:bucket.log' ],
20
+ [ :output_file, '-o', '/var/log/s3/:year/:month/:bucket.log' ],
21
+
22
+ [ :cache_dir, '--cache-dir', '/var/run/s3_cache/:bucket/:year/:month/:day' ],
23
+ [ :cache_dir, '-x', '/var/run/s3_cache/:bucket/:year/:month/:day' ],
24
+
25
+ [ :list, '--list', true ],
26
+ [ :list, '-l', true ],
27
+
28
+ [ :debug, '--debug', true ],
29
+ [ :debug, '-d', true ],
30
+ [ :debug, '--debug', 'aws' ],
31
+ [ :debug, '-d', 'aws' ],
32
+
33
+ [ :config_file, '--config-file', '/my/config/file.conf' ],
34
+ [ :config_file, '-c', '/my/config/file.conf' ],
35
+ [ :config_file, '--config-file', '' ],
36
+ [ :config_file, '-c', '' ],
37
+
38
+ # :rename_bucket_keys => [ '-m', '--rename-bucket-keys', nil ],
39
+ ]
40
+ end
41
+
42
+ it "should show help message" do
43
+ output = StringIO.new
44
+ options = Ralf::OptionParser.parse('-h'.split, output)
45
+ options.should be_nil
46
+ output.string.should_not be_empty
47
+ output.string.should include("Show this message")
48
+ end
49
+
50
+ it "should show version" do
51
+ output = StringIO.new
52
+ File.should_receive(:read).and_return('1.2.3')
53
+ options = Ralf::OptionParser.parse('-v'.split, output)
54
+ options.should be_nil
55
+ output.string.should_not be_empty
56
+ output.string.should include("1.2.3")
57
+ end
58
+
59
+ it "should parse all options short or long" do
60
+ output = StringIO.new
61
+
62
+ @valid_arguments.to_a.each do |argument_spec|
63
+ options = to_argv_array(argument_spec)
64
+ config = Ralf::OptionParser.parse(options, output)
65
+
66
+ config.should have_key(argument_spec.first)
67
+ config[argument_spec.first].should eql(argument_spec.last)
68
+
69
+ output.string.should be_empty
70
+ end
71
+
72
+ end
73
+
74
+ it "should allow two dates for range" do
75
+ output = StringIO.new
76
+
77
+ options = Ralf::OptionParser.parse("--range yesterday,today".split, output)
78
+
79
+ options.should have_key(:range)
80
+ options[:range].should eql(['yesterday', 'today'])
81
+ end
82
+
83
+ it "should produce error for missing argument" do
84
+ output = StringIO.new
85
+
86
+ lambda {
87
+ options = Ralf::OptionParser.parse("--range".split, output)
88
+ }.should raise_error(OptionParser::MissingArgument)
89
+ end
90
+
91
+ private
92
+
93
+ def to_argv_array(spec)
94
+ spec[1..-1].delete_if{ |v| true == v }.map{ |v| v.is_a?(Array) ? v.join(',') : v }
95
+ end
96
+
97
+ end