ralf 0.1.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/README.rdoc +126 -0
- data/Rakefile +57 -0
- data/VERSION +1 -0
- data/bin/ralf +41 -0
- data/lib/ralf/bucket.rb +66 -0
- data/lib/ralf/config.rb +176 -0
- data/lib/ralf/interpolation.rb +48 -0
- data/lib/ralf/log.rb +18 -0
- data/lib/ralf/option_parser.rb +149 -0
- data/lib/ralf.rb +173 -0
- data/spec/fixtures/apache.log +7 -0
- data/spec/fixtures/example_buckets.yaml +48 -0
- data/spec/ralf/bucket_spec.rb +119 -0
- data/spec/ralf/config_spec.rb +131 -0
- data/spec/ralf/interpolation_spec.rb +45 -0
- data/spec/ralf/log_spec.rb +63 -0
- data/spec/ralf/option_parser_spec.rb +97 -0
- data/spec/ralf_spec.rb +274 -206
- data/spec/spec.opts +5 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/support/fakeweb.rb +4 -0
- metadata +105 -27
data/lib/ralf.rb
ADDED
@@ -0,0 +1,173 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'right_aws'
|
3
|
+
require 'logmerge'
|
4
|
+
require 'ftools'
|
5
|
+
require 'ralf/config'
|
6
|
+
require 'ralf/bucket'
|
7
|
+
require 'chronic'
|
8
|
+
|
9
|
+
class Ralf
|
10
|
+
|
11
|
+
private
|
12
|
+
RLIMIT_NOFILE_HEADROOM = 100 # number of file descriptors to allocate above number of logfiles
|
13
|
+
|
14
|
+
public
|
15
|
+
|
16
|
+
ROOT_DEFAULT_CONFIG_FILE = '/etc/ralf.conf'
|
17
|
+
USER_DEFAULT_CONFIG_FILE = '~/.ralf.conf'
|
18
|
+
|
19
|
+
AMAZON_LOG_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
|
20
|
+
|
21
|
+
# The current configuration.
|
22
|
+
attr_reader :config
|
23
|
+
|
24
|
+
# Instance of RightAws::S3 used by Ralf to talk to Amazon S3.
|
25
|
+
attr_reader :s3
|
26
|
+
|
27
|
+
# Create instance and run with the specified parameters
|
28
|
+
def self.run(options)
|
29
|
+
list = options.delete(:list)
|
30
|
+
|
31
|
+
ralf = Ralf.new(options)
|
32
|
+
if list
|
33
|
+
ralf.list
|
34
|
+
else
|
35
|
+
ralf.run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Create new Ralf instance
|
40
|
+
def initialize(options = {})
|
41
|
+
initial_options = options.dup
|
42
|
+
|
43
|
+
@config = load_config(initial_options.delete(:config_file))
|
44
|
+
|
45
|
+
config.merge!(initial_options)
|
46
|
+
config.validate!
|
47
|
+
|
48
|
+
RightAws::RightAwsBaseInterface.caching = true # enable caching to speed up
|
49
|
+
Bucket.s3 = RightAws::S3.new(config.aws_access_key_id, config.aws_secret_access_key,
|
50
|
+
:protocol => 'http', :port => 80,
|
51
|
+
:logger => Logger.new('aws' == config.debug? ? $stdout : StringIO.new)
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
# For all buckets for all dates in the configured range download the available
|
56
|
+
# log files. After downloading, merge the log files and convert the format
|
57
|
+
# from Amazon Log Format to Apache Common Log Format.
|
58
|
+
def run(output_file = nil)
|
59
|
+
config.output_file = output_file unless output_file.nil?
|
60
|
+
|
61
|
+
raise ArgumentError.new("--output-file required") if config.output_file_missing?
|
62
|
+
raise ArgumentError.new("--output-file requires ':bucket' variable") if (config.buckets.nil? || config.buckets.size > 1) and !(config.output_file_format =~ /:bucket/)
|
63
|
+
|
64
|
+
puts "Processing range #{config.range}" if config.debug?
|
65
|
+
|
66
|
+
# iterate over all buckets
|
67
|
+
Bucket.each(config.buckets) do |bucket|
|
68
|
+
|
69
|
+
print "#{bucket.name}: " if config.debug?
|
70
|
+
|
71
|
+
# iterate over the full range
|
72
|
+
log_files = []
|
73
|
+
config.range.each do |date|
|
74
|
+
dir = config.cache_dir(:bucket => bucket.name, :date => date)
|
75
|
+
log_files << Ralf.download_logs(bucket, date, dir)
|
76
|
+
end
|
77
|
+
log_files.flatten!
|
78
|
+
|
79
|
+
# determine output file names
|
80
|
+
output_log = config.output_file(:date => config.range.end, :bucket => bucket.name)
|
81
|
+
merged_log = output_log + ".alf"
|
82
|
+
|
83
|
+
# create directory for output file
|
84
|
+
File.makedirs(File.dirname(merged_log))
|
85
|
+
|
86
|
+
# merge the log files
|
87
|
+
Ralf.merge(merged_log, log_files)
|
88
|
+
|
89
|
+
# convert to common log format
|
90
|
+
Ralf.convert_to_common_log_format(merged_log, output_log)
|
91
|
+
|
92
|
+
puts "#{log_files.size} files" if config.debug?
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# List all buckets with the logging info.
|
97
|
+
def list(with_logging = false)
|
98
|
+
puts "Listing buckets..." if config.debug?
|
99
|
+
|
100
|
+
Bucket.each(config.buckets, with_logging) do |bucket|
|
101
|
+
print "#{bucket.name}"
|
102
|
+
puts bucket.logging_enabled? ? " [#{bucket.targetbucket}/#{bucket.targetprefix}]" : " [-]"
|
103
|
+
end
|
104
|
+
|
105
|
+
nil
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
# Download log files for +bucket+ and +date+ to +dir+.
|
111
|
+
def self.download_logs(bucket, date, dir)
|
112
|
+
File.makedirs(dir)
|
113
|
+
|
114
|
+
# iterate over the available log files, saving them to disk and
|
115
|
+
log_files = []
|
116
|
+
bucket.each_log(date) do |log|
|
117
|
+
log_files << log.save_to_dir(dir)
|
118
|
+
end
|
119
|
+
log_files
|
120
|
+
end
|
121
|
+
|
122
|
+
# Takes an array of log file names and merges them on ascending timestamp
|
123
|
+
# into +output_file+ name. Assumes the +log_files+ are sorted
|
124
|
+
# on ascending timestamp.
|
125
|
+
def self.merge(output_file, log_files)
|
126
|
+
update_rlimit_nofile(log_files.size)
|
127
|
+
File.open(output_file, 'w') do |out_file|
|
128
|
+
LogMerge::Merger.merge out_file, *log_files
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# Convert the input_log file to Apache Common Log Format into output_log
|
133
|
+
def self.convert_to_common_log_format(input_log, output_log)
|
134
|
+
out_file = File.open(output_log, 'w')
|
135
|
+
File.open(input_log, 'r') do |in_file|
|
136
|
+
while (line = in_file.gets)
|
137
|
+
out_file.puts(translate_to_clf(line))
|
138
|
+
end
|
139
|
+
end
|
140
|
+
out_file.close
|
141
|
+
end
|
142
|
+
|
143
|
+
def self.translate_to_clf(line)
|
144
|
+
if line =~ AMAZON_LOG_FORMAT
|
145
|
+
# host, date, ip, acl, request, status, bytes, agent = $2, $3, $4, $5, $9, $10, $12, $17
|
146
|
+
"%s - %s [%s] \"%s\" %d %s \"%s\" \"%s\"" % [$4, $5, $3, $9, $10, $12, $16, $17]
|
147
|
+
else
|
148
|
+
$stderr.puts "# ERROR: #{line}"
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def load_config(cli_config_file)
|
153
|
+
result = nil
|
154
|
+
if cli_config_file
|
155
|
+
result = Ralf::Config.load_file(cli_config_file) unless cli_config_file.empty?
|
156
|
+
else
|
157
|
+
config_file = (0 == Process.uid ? ROOT_DEFAULT_CONFIG_FILE : File.expand_path(USER_DEFAULT_CONFIG_FILE))
|
158
|
+
result = Ralf::Config.load_file(config_file) if File.exist?(config_file)
|
159
|
+
end
|
160
|
+
result || Ralf::Config.new
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.update_rlimit_nofile(number_of_files)
|
164
|
+
new_rlimit_nofile = number_of_files + RLIMIT_NOFILE_HEADROOM
|
165
|
+
|
166
|
+
# getrlimit returns array with soft and hard limit [soft, hard]
|
167
|
+
rlimit_nofile = Process::getrlimit(Process::RLIMIT_NOFILE)
|
168
|
+
if new_rlimit_nofile > rlimit_nofile.first
|
169
|
+
Process.setrlimit(Process::RLIMIT_NOFILE, new_rlimit_nofile) rescue nil
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
10.34.211.35 - F7CECCD7A21B4B96 [10/Feb/2010:07:16:19 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 456 - "-" "Jakarta Commons-HttpClient/3.0" -
|
2
|
+
10.34.211.35 - E415BCEA05AFB84F [10/Feb/2010:07:16:19 +0000] "POST /soap/ HTTP/1.1" 200 - 797 1003 74 41 "-" "Axis/1.3" -
|
3
|
+
10.36.142.18 - BE89D3B6953D719E [10/Feb/2010:07:16:28 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 433 - "-" "Jakarta Commons-HttpClient/3.0" -
|
4
|
+
10.36.142.18 - E15F489437E852EA [10/Feb/2010:07:16:28 +0000] "POST /soap/ HTTP/1.1" 200 - 797 954 36 14 "-" "Axis/1.3" -
|
5
|
+
10.32.219.38 - 784FD457838EFF42 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -
|
6
|
+
10.32.219.38 - 6E239BC5A4AC757C [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -
|
7
|
+
10.217.37.15 - 0B76C90B3634290B [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -
|
@@ -0,0 +1,48 @@
|
|
1
|
+
---
|
2
|
+
-
|
3
|
+
:name: 'test1'
|
4
|
+
:logging_info:
|
5
|
+
:enabled: true
|
6
|
+
:targetbucket: test1
|
7
|
+
:targetprefix: logs/
|
8
|
+
:keys:
|
9
|
+
-
|
10
|
+
:name: 'logs/2010-02-10-00-05-32-ZDRFGTCKUYVJCT'
|
11
|
+
-
|
12
|
+
:name: 'logs/2010-02-11-00-05-32-ZDRFGTCKUYVJCT'
|
13
|
+
-
|
14
|
+
:name: 'test2'
|
15
|
+
:logging_info:
|
16
|
+
:enabled: true
|
17
|
+
:targetbucket: target_test2
|
18
|
+
:targetprefix: logs/access_log-
|
19
|
+
:keys:
|
20
|
+
-
|
21
|
+
:name: 'logs/access_log-2010-02-12-00-05-32-ZDRFGTCKUYVJCT'
|
22
|
+
-
|
23
|
+
:name: 'logs/access_log-2010-02-13-00-05-32-ZDRFGTCKUYVJCT'
|
24
|
+
-
|
25
|
+
:name: 'test3'
|
26
|
+
:logging_info:
|
27
|
+
:enabled: false
|
28
|
+
:targetbucket: ''
|
29
|
+
:targetprefix: ''
|
30
|
+
:keys: []
|
31
|
+
-
|
32
|
+
:name: 'test4'
|
33
|
+
:logging_info:
|
34
|
+
:enabled: true
|
35
|
+
:targetbucket: test4
|
36
|
+
:targetprefix: logs/
|
37
|
+
:keys:
|
38
|
+
-
|
39
|
+
:name: 'logs/2010-02-14-00-05-32-ZDRFGTCKUYVJCT'
|
40
|
+
-
|
41
|
+
:name: 'logs/2010-02-15-00-05-32-ZDRFGTCKUYVJCT'
|
42
|
+
-
|
43
|
+
:name: 'target_test2'
|
44
|
+
:logging_info:
|
45
|
+
:enabled: false
|
46
|
+
:targetbucket: ''
|
47
|
+
:targetprefix: ''
|
48
|
+
:keys: []
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require 'ralf'
|
4
|
+
require 'ralf/bucket'
|
5
|
+
|
6
|
+
describe Ralf::Bucket do
|
7
|
+
|
8
|
+
before(:all) do
|
9
|
+
@valid_logging_info = {
|
10
|
+
:enabled => true,
|
11
|
+
:targetbucket => 'targetbucket',
|
12
|
+
:targetprefix => 'logs/',
|
13
|
+
}
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should initialize properly" do
|
17
|
+
mock_S3
|
18
|
+
|
19
|
+
name = 's3_bucket'
|
20
|
+
logging_info = {
|
21
|
+
:enabled => true,
|
22
|
+
:targetbucket => 's3_bucket',
|
23
|
+
:targetprefix => 'logs/',
|
24
|
+
}
|
25
|
+
bucket = mock(name)
|
26
|
+
bucket.should_receive(:logging_info).and_return(logging_info)
|
27
|
+
bucket.should_receive(:name).and_return(name)
|
28
|
+
|
29
|
+
lambda {
|
30
|
+
Ralf::Bucket.new(bucket)
|
31
|
+
}.should_not raise_error
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should set targetbucket to bucket returned by logging_info" do
|
35
|
+
mock_S3
|
36
|
+
|
37
|
+
name = 's3_bucket'
|
38
|
+
targetbucket_name = 's3_targetbucket'
|
39
|
+
logging_info = {
|
40
|
+
:enabled => true,
|
41
|
+
:targetbucket => targetbucket_name,
|
42
|
+
:targetprefix => 'logs/',
|
43
|
+
}
|
44
|
+
bucket = mock(name)
|
45
|
+
bucket.should_receive(:logging_info).and_return(logging_info)
|
46
|
+
bucket.should_receive(:name).and_return(name)
|
47
|
+
|
48
|
+
Ralf::Bucket.new(bucket)
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should support iteration over all buckets" do
|
52
|
+
mock_S3
|
53
|
+
|
54
|
+
yielded_buckets = []
|
55
|
+
Ralf::Bucket.each do |bucket|
|
56
|
+
yielded_buckets << bucket
|
57
|
+
end
|
58
|
+
yielded_buckets.should have(@enabled_buckets_count).items
|
59
|
+
yielded_buckets.each { |bucket| bucket.name.should_not be_nil }
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should support iteration over specific buckets" do
|
63
|
+
mock_S3
|
64
|
+
|
65
|
+
yielded_buckets = []
|
66
|
+
Ralf::Bucket.each(@example_buckets.keys) do |bucket|
|
67
|
+
yielded_buckets << bucket
|
68
|
+
end
|
69
|
+
yielded_buckets.should have(@enabled_buckets_count).items
|
70
|
+
yielded_buckets.each do |bucket|
|
71
|
+
bucket.logging_enabled?.should eql(@example_buckets[bucket.name].logging_info[:enabled])
|
72
|
+
bucket.targetbucket.should eql(@example_buckets[bucket.name].logging_info[:targetbucket])
|
73
|
+
bucket.targetprefix.should eql(@example_buckets[bucket.name].logging_info[:targetprefix])
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should support iterating over all logs in a bucket" do
|
78
|
+
mock_S3
|
79
|
+
|
80
|
+
bucket_mock = @example_buckets['test1']
|
81
|
+
key1, key2 = mock('key1'), mock('key2')
|
82
|
+
keys = [key1, key2]
|
83
|
+
bucket_mock.should_receive(:keys).with(:prefix => 'logs/2010-05-17').and_return(keys)
|
84
|
+
|
85
|
+
expected_logs = []
|
86
|
+
keys.each do |key|
|
87
|
+
log = mock('Log')
|
88
|
+
Ralf::Log.should_receive(:new).with(key, @example_buckets['test1'].logging_info[:targetprefix]).and_return(log)
|
89
|
+
expected_logs << log
|
90
|
+
end
|
91
|
+
|
92
|
+
bucket = Ralf::Bucket.new(bucket_mock)
|
93
|
+
|
94
|
+
yielded_logs = []
|
95
|
+
bucket.each_log(Date.new(2010,05,17)) do |log|
|
96
|
+
yielded_logs << log
|
97
|
+
end
|
98
|
+
yielded_logs.should have(keys.size).items
|
99
|
+
yielded_logs.should eql(expected_logs)
|
100
|
+
end
|
101
|
+
|
102
|
+
def mock_S3
|
103
|
+
@s3_mock = mock('S3')
|
104
|
+
Ralf::Bucket.s3 = @s3_mock
|
105
|
+
|
106
|
+
# load example buckets (2 disabled)
|
107
|
+
@example_buckets = load_example_bucket_mocks
|
108
|
+
@enabled_buckets_count = @example_buckets.size - 2
|
109
|
+
|
110
|
+
# make s3_mock return individual buckets
|
111
|
+
@s3_mock.should_receive(:bucket).any_number_of_times do |name|
|
112
|
+
@example_buckets[name]
|
113
|
+
end
|
114
|
+
|
115
|
+
# make s3_mock return all buckets
|
116
|
+
@s3_mock.should_receive(:buckets).any_number_of_times.and_return(@example_buckets.values)
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require 'ralf'
|
4
|
+
require 'ralf/config'
|
5
|
+
|
6
|
+
describe Ralf::Config do
|
7
|
+
|
8
|
+
before(:all) do
|
9
|
+
ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = nil, nil
|
10
|
+
|
11
|
+
@aws_credentials = {
|
12
|
+
:aws_access_key_id => 'access_key_id',
|
13
|
+
:aws_secret_access_key => 'secret_key',
|
14
|
+
}
|
15
|
+
|
16
|
+
@valid_options = {
|
17
|
+
:output_file => 'my_log.log',
|
18
|
+
}.merge(@aws_credentials)
|
19
|
+
|
20
|
+
@date = Date.strptime('2010-02-09')
|
21
|
+
@bucket = 'my.bucket.org'
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should initialize properly" do
|
25
|
+
config = Ralf::Config.new
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should raise error for missing credentials" do
|
29
|
+
lambda {
|
30
|
+
config = Ralf::Config.new
|
31
|
+
config.validate!
|
32
|
+
}.should raise_error(Ralf::Config::ConfigurationError, 'aws_access_key_id missing, aws_secret_access_key missing')
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should handle range assignment" do
|
36
|
+
now = Time.now
|
37
|
+
yesterday = Date.today - 1
|
38
|
+
Time.should_receive(:now).any_number_of_times.and_return(now)
|
39
|
+
config = Ralf::Config.new(@valid_options)
|
40
|
+
config.range = 'yesterday'
|
41
|
+
config.range.should eql(Range.new(yesterday, yesterday))
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should interpolate date (:year, :month, :day) variables in output_file" do
|
45
|
+
config = Ralf::Config.new(@valid_options.merge(:output_file => ':year/:month/access.log'))
|
46
|
+
config.output_file(:date => @date).should eql('2010/02/access.log')
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should interpolate :bucket variable in output_file" do
|
50
|
+
config = Ralf::Config.new(@valid_options.merge(:output_file => ':bucket.log'))
|
51
|
+
config.output_file(:date => @date, :bucket => @bucket).should eql('my.bucket.org.log')
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should interpolate variables in cache_dir" do
|
55
|
+
config = Ralf::Config.new(@valid_options.merge(:cache_dir => ':year/:month/:bucket'))
|
56
|
+
config.cache_dir(:date => @date, :bucket => @bucket).should eql('2010/02/my.bucket.org')
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should allow 'this month' with base 'yesterday'" do
|
60
|
+
Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
|
61
|
+
config = Ralf::Config.new(:range => 'this month', :now => 'yesterday')
|
62
|
+
config.range.to_s.should eql('2010-04-01..2010-04-30')
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should merge options" do
|
66
|
+
Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
|
67
|
+
config = Ralf::Config.new
|
68
|
+
config.merge!(:range => 'this month', :now => 'yesterday')
|
69
|
+
config.range.to_s.should eql('2010-04-01..2010-04-30')
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should support setting now after range and recompute range" do
|
73
|
+
Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
|
74
|
+
config = Ralf::Config.new
|
75
|
+
config.merge!(:range => 'this month')
|
76
|
+
config.merge!(:now => 'yesterday')
|
77
|
+
config.range.to_s.should eql('2010-04-01..2010-04-30')
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should support setting range first then change now (1st day of month)" do
|
81
|
+
Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
|
82
|
+
config = Ralf::Config.new
|
83
|
+
config.merge!(:range => 'this month')
|
84
|
+
config.range.to_s.should eql('2010-05-01..2010-05-01')
|
85
|
+
config.merge!(:now => 'yesterday')
|
86
|
+
config.range.to_s.should eql('2010-04-01..2010-04-30')
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should support setting range first then change now" do
|
90
|
+
Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 08 16:31:00 +0100 2010'))
|
91
|
+
config = Ralf::Config.new
|
92
|
+
config.merge!(:range => 'this month')
|
93
|
+
config.range.to_s.should eql('2010-05-01..2010-05-07')
|
94
|
+
config.merge!(:now => '2010-05-06')
|
95
|
+
config.range.to_s.should eql('2010-05-01..2010-05-06')
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should set default cache_dir to '/var/log/ralf/:bucket' when running as root" do
|
99
|
+
Process.should_receive(:uid).and_return(0)
|
100
|
+
config = Ralf::Config.new
|
101
|
+
config.cache_dir_format.should eql('/var/log/ralf/:bucket')
|
102
|
+
end
|
103
|
+
|
104
|
+
it "should set default cache_dir to '~/.ralf/:bucket' when running as regular user" do
|
105
|
+
Process.should_receive(:uid).and_return(1)
|
106
|
+
File.should_receive(:expand_path).with('~/.ralf/:bucket').and_return('/Users/me/.ralf/:bucket')
|
107
|
+
config = Ralf::Config.new
|
108
|
+
config.cache_dir_format.should eql('/Users/me/.ralf/:bucket')
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should allow overriding bucket with merge!" do
|
112
|
+
config = Ralf::Config.new(:buckets => ['bucket1', 'bucket2'])
|
113
|
+
config.buckets = ['bucket3']
|
114
|
+
config.buckets.should eql(['bucket3'])
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should load configuration from file" do
|
118
|
+
YAML.should_receive(:load_file).with('my_config.yaml').and_return(@valid_options)
|
119
|
+
config = Ralf::Config.load_file('my_config.yaml')
|
120
|
+
config.output_file_format.should eql(@valid_options[:output_file])
|
121
|
+
end
|
122
|
+
|
123
|
+
it "should warn and continue (not raise exception) when unknown configuration variables are set" do
|
124
|
+
$stdout = StringIO.new
|
125
|
+
lambda {
|
126
|
+
config = Ralf::Config.new(:unknown_variable => 100)
|
127
|
+
}.should_not raise_error
|
128
|
+
$stdout.string.should eql("Warning: invalid configuration variable: unknown_variable\n")
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require 'ralf/interpolation'
|
4
|
+
|
5
|
+
describe Ralf::Interpolation do
|
6
|
+
before(:all) do
|
7
|
+
@bucket = 'bucket.mybuckets.org'
|
8
|
+
@date = Date.strptime('2010-02-09')
|
9
|
+
end
|
10
|
+
|
11
|
+
[
|
12
|
+
[':year', '2010'],
|
13
|
+
[':month', '02'],
|
14
|
+
[':day', '09'],
|
15
|
+
[':week', '06']
|
16
|
+
].each do |match, result, bucket|
|
17
|
+
it "should replace '#{match}' with '#{result}'" do
|
18
|
+
Ralf::Interpolation.interpolate(match, :date => @date).should eql(result)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should raise an error when not all symbols could be interpolated" do
|
23
|
+
lambda {
|
24
|
+
Ralf::Interpolation.interpolate(':unknown', :date => @date)
|
25
|
+
}.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should interpolate :bucket" do
|
29
|
+
Ralf::Interpolation.interpolate(':year/:month/:day/:bucket.log',
|
30
|
+
:date => @date, :bucket => @bucket).should eql('2010/02/09/bucket.mybuckets.org.log')
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should require :bucket interpolation if bucket specified" do
|
34
|
+
lambda {
|
35
|
+
Ralf::Interpolation.interpolate(':year/:month/:day.log', {:date => @date, :bucket => @bucket}, [:bucket])
|
36
|
+
}.should raise_error(Ralf::Interpolation::VariableMissing, ':bucket variable missing')
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should raise an error when :bucket can not be interpolated" do
|
40
|
+
lambda {
|
41
|
+
Ralf::Interpolation.interpolate(':bucket', :date => @date)
|
42
|
+
}.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require 'ralf'
|
4
|
+
require 'ralf/log'
|
5
|
+
|
6
|
+
describe Ralf::Log do
|
7
|
+
|
8
|
+
it "should initialize properly" do
|
9
|
+
key = mock('key')
|
10
|
+
prefix = mock('prefix')
|
11
|
+
lambda {
|
12
|
+
Ralf::Log.new(key, prefix)
|
13
|
+
}.should_not raise_error
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should remove prefix when returing name" do
|
17
|
+
key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
|
18
|
+
log = Ralf::Log.new(key, 'log/access_log-')
|
19
|
+
log.name.should eql('2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should save specified file to dir" do
|
23
|
+
key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
|
24
|
+
log = Ralf::Log.new(key, 'log/access_log-')
|
25
|
+
|
26
|
+
dir = '/var/log/s3'
|
27
|
+
filename = File.join(dir, log.name)
|
28
|
+
fileio = mock('File')
|
29
|
+
key.should_receive(:data).and_return('testdata')
|
30
|
+
File.should_receive(:open).with(filename, 'w').and_yield(fileio)
|
31
|
+
File.should_receive(:exist?).with(filename).and_return(false)
|
32
|
+
fileio.should_receive(:write).with('testdata')
|
33
|
+
log.save_to_dir(dir)
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should not save file if it exists and caching enabled (default)" do
|
37
|
+
key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
|
38
|
+
log = Ralf::Log.new(key, 'log/access_log-')
|
39
|
+
|
40
|
+
dir = '/var/log/s3'
|
41
|
+
filename = File.join(dir, log.name)
|
42
|
+
fileio = mock('File')
|
43
|
+
File.should_receive(:exist?).with(filename).and_return(true)
|
44
|
+
File.should_not_receive(:open).with(filename, 'w')
|
45
|
+
log.save_to_dir(dir)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should not check file if use_cache = false" do
|
49
|
+
key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
|
50
|
+
log = Ralf::Log.new(key, 'log/access_log-')
|
51
|
+
|
52
|
+
dir = '/var/log/s3'
|
53
|
+
filename = File.join(dir, log.name)
|
54
|
+
fileio = mock('File')
|
55
|
+
|
56
|
+
key.should_receive(:data).and_return('testdata')
|
57
|
+
File.should_receive(:open).with(filename, 'w').and_yield(fileio)
|
58
|
+
File.should_not_receive(:exist?).with(filename)
|
59
|
+
fileio.should_receive(:write).with('testdata')
|
60
|
+
log.save_to_dir(dir, false)
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require 'ralf'
|
4
|
+
require 'ralf/option_parser'
|
5
|
+
|
6
|
+
describe Ralf::OptionParser do
|
7
|
+
|
8
|
+
before(:all) do
|
9
|
+
@valid_arguments = [
|
10
|
+
[ :buckets, '--buckets', [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
|
11
|
+
[ :buckets, '-b', [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
|
12
|
+
|
13
|
+
[ :range, '--range', ['today'] ],
|
14
|
+
[ :range, '-r', ['today'] ],
|
15
|
+
|
16
|
+
[ :now, '--now', 'yesterday' ],
|
17
|
+
[ :now, '-t', 'yesterday' ],
|
18
|
+
|
19
|
+
[ :output_file, '--output-file', '/var/log/s3/:year/:month/:bucket.log' ],
|
20
|
+
[ :output_file, '-o', '/var/log/s3/:year/:month/:bucket.log' ],
|
21
|
+
|
22
|
+
[ :cache_dir, '--cache-dir', '/var/run/s3_cache/:bucket/:year/:month/:day' ],
|
23
|
+
[ :cache_dir, '-x', '/var/run/s3_cache/:bucket/:year/:month/:day' ],
|
24
|
+
|
25
|
+
[ :list, '--list', true ],
|
26
|
+
[ :list, '-l', true ],
|
27
|
+
|
28
|
+
[ :debug, '--debug', true ],
|
29
|
+
[ :debug, '-d', true ],
|
30
|
+
[ :debug, '--debug', 'aws' ],
|
31
|
+
[ :debug, '-d', 'aws' ],
|
32
|
+
|
33
|
+
[ :config_file, '--config-file', '/my/config/file.conf' ],
|
34
|
+
[ :config_file, '-c', '/my/config/file.conf' ],
|
35
|
+
[ :config_file, '--config-file', '' ],
|
36
|
+
[ :config_file, '-c', '' ],
|
37
|
+
|
38
|
+
# :rename_bucket_keys => [ '-m', '--rename-bucket-keys', nil ],
|
39
|
+
]
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should show help message" do
|
43
|
+
output = StringIO.new
|
44
|
+
options = Ralf::OptionParser.parse('-h'.split, output)
|
45
|
+
options.should be_nil
|
46
|
+
output.string.should_not be_empty
|
47
|
+
output.string.should include("Show this message")
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should show version" do
|
51
|
+
output = StringIO.new
|
52
|
+
File.should_receive(:read).and_return('1.2.3')
|
53
|
+
options = Ralf::OptionParser.parse('-v'.split, output)
|
54
|
+
options.should be_nil
|
55
|
+
output.string.should_not be_empty
|
56
|
+
output.string.should include("1.2.3")
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should parse all options short or long" do
|
60
|
+
output = StringIO.new
|
61
|
+
|
62
|
+
@valid_arguments.to_a.each do |argument_spec|
|
63
|
+
options = to_argv_array(argument_spec)
|
64
|
+
config = Ralf::OptionParser.parse(options, output)
|
65
|
+
|
66
|
+
config.should have_key(argument_spec.first)
|
67
|
+
config[argument_spec.first].should eql(argument_spec.last)
|
68
|
+
|
69
|
+
output.string.should be_empty
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should allow two dates for range" do
|
75
|
+
output = StringIO.new
|
76
|
+
|
77
|
+
options = Ralf::OptionParser.parse("--range yesterday,today".split, output)
|
78
|
+
|
79
|
+
options.should have_key(:range)
|
80
|
+
options[:range].should eql(['yesterday', 'today'])
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should produce error for missing argument" do
|
84
|
+
output = StringIO.new
|
85
|
+
|
86
|
+
lambda {
|
87
|
+
options = Ralf::OptionParser.parse("--range".split, output)
|
88
|
+
}.should raise_error(OptionParser::MissingArgument)
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def to_argv_array(spec)
|
94
|
+
spec[1..-1].delete_if{ |v| true == v }.map{ |v| v.is_a?(Array) ? v.join(',') : v }
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|