cwlogs-s3 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +4 -1
- data/cwlogs-s3.gemspec +1 -1
- data/lib/cwlogs-s3/command.rb +5 -4
- metadata +1 -1
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# cwlogs-s3
|
|
2
2
|
|
|
3
|
+
**[Click here](http://hipsterdevblog.com/blog/2015/02/24/part-1-exporting-and-analysing-cloudwatch-logs-with-data-pipeline-and-emr/) for
|
|
4
|
+
a blog post on how to use this tool with AWS Data Pipeline.**
|
|
5
|
+
|
|
3
6
|
Task for exporting CloudWatch logs to S3. Useful for then running logs through EMR for analysis. Designed to be run
|
|
4
7
|
via AWS Data Pipeline.
|
|
5
8
|
|
|
@@ -30,7 +33,7 @@ OPTIONS
|
|
|
30
33
|
(e.g. INFO, DEBUG) (default: WARN)
|
|
31
34
|
--error-output-format <s - The format to use when outputting errors (e.g. b
|
|
32
35
|
> asic, advanced) (default: basic)
|
|
33
|
-
--help -h - Show this
|
|
36
|
+
--help -h - Show this message
|
|
34
37
|
```
|
|
35
38
|
|
|
36
39
|
For example:
|
data/cwlogs-s3.gemspec
CHANGED
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "cwlogs-s3"
|
|
7
|
-
spec.version = '0.0.
|
|
7
|
+
spec.version = '0.0.5'
|
|
8
8
|
spec.authors = ["Tim-B"]
|
|
9
9
|
spec.email = ["tim@galacticcode.com"]
|
|
10
10
|
spec.summary = %q{Exports logs from CloudWatch logs and uploads them to S3}
|
data/lib/cwlogs-s3/command.rb
CHANGED
|
@@ -19,8 +19,8 @@ module CWLogsToS3
|
|
|
19
19
|
start = Time.at(ending.to_i - period)
|
|
20
20
|
|
|
21
21
|
s3Uri = URI(command_options[:s3_path])
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
bucket = s3Uri.host
|
|
23
|
+
s3_prefix = s3Uri.path
|
|
24
24
|
|
|
25
25
|
Escort::Logger.output.puts "Exporting from #{start} to #{ending}"
|
|
26
26
|
|
|
@@ -50,6 +50,7 @@ module CWLogsToS3
|
|
|
50
50
|
log_stream_name: stream,
|
|
51
51
|
start_time: start.to_i * 1000,
|
|
52
52
|
end_time: ending.to_i * 1000,
|
|
53
|
+
start_from_head: true
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
resp.each do |log_page|
|
|
@@ -69,8 +70,8 @@ module CWLogsToS3
|
|
|
69
70
|
@event_cnt = @event_cnt + 1
|
|
70
71
|
page_content << event[:message] << "\n"
|
|
71
72
|
end
|
|
72
|
-
object_name =
|
|
73
|
-
@s3.bucket(
|
|
73
|
+
object_name = command_options[:prefix] + randomise_prefix + '_' + @page_cnt.to_s + '.log'
|
|
74
|
+
@s3.bucket(command_options[:bucket]).object(object_name).put(
|
|
74
75
|
:body => page_content
|
|
75
76
|
)
|
|
76
77
|
Escort::Logger.output.puts "Put #{object_name} to S3."
|