cwlogs-s3 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +4 -1
- data/cwlogs-s3.gemspec +1 -1
- data/lib/cwlogs-s3/command.rb +5 -4
- metadata +1 -1
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# cwlogs-s3
|
|
2
2
|
|
|
3
|
+
**[Click here](http://hipsterdevblog.com/blog/2015/02/24/part-1-exporting-and-analysing-cloudwatch-logs-with-data-pipeline-and-emr/) for
|
|
4
|
+
a blog post on how to use this tool with AWS Data Pipeline.**
|
|
5
|
+
|
|
3
6
|
Task for exporting CloudWatch logs to S3. Useful for then running logs through EMR for analysis. Designed to be run
|
|
4
7
|
via AWS Data Pipeline.
|
|
5
8
|
|
|
@@ -30,7 +33,7 @@ OPTIONS
|
|
|
30
33
|
(e.g. INFO, DEBUG) (default: WARN)
|
|
31
34
|
--error-output-format <s - The format to use when outputting errors (e.g. b
|
|
32
35
|
> asic, advanced) (default: basic)
|
|
33
|
-
--help -h - Show this
|
|
36
|
+
--help -h - Show this message
|
|
34
37
|
```
|
|
35
38
|
|
|
36
39
|
For example:
|
data/cwlogs-s3.gemspec
CHANGED
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "cwlogs-s3"
|
|
7
|
-
spec.version = '0.0.
|
|
7
|
+
spec.version = '0.0.5'
|
|
8
8
|
spec.authors = ["Tim-B"]
|
|
9
9
|
spec.email = ["tim@galacticcode.com"]
|
|
10
10
|
spec.summary = %q{Exports logs from CloudWatch logs and uploads them to S3}
|
data/lib/cwlogs-s3/command.rb
CHANGED
|
@@ -19,8 +19,8 @@ module CWLogsToS3
|
|
|
19
19
|
start = Time.at(ending.to_i - period)
|
|
20
20
|
|
|
21
21
|
s3Uri = URI(command_options[:s3_path])
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
bucket = s3Uri.host
|
|
23
|
+
s3_prefix = s3Uri.path
|
|
24
24
|
|
|
25
25
|
Escort::Logger.output.puts "Exporting from #{start} to #{ending}"
|
|
26
26
|
|
|
@@ -50,6 +50,7 @@ module CWLogsToS3
|
|
|
50
50
|
log_stream_name: stream,
|
|
51
51
|
start_time: start.to_i * 1000,
|
|
52
52
|
end_time: ending.to_i * 1000,
|
|
53
|
+
start_from_head: true
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
resp.each do |log_page|
|
|
@@ -69,8 +70,8 @@ module CWLogsToS3
|
|
|
69
70
|
@event_cnt = @event_cnt + 1
|
|
70
71
|
page_content << event[:message] << "\n"
|
|
71
72
|
end
|
|
72
|
-
object_name =
|
|
73
|
-
@s3.bucket(
|
|
73
|
+
object_name = command_options[:prefix] + randomise_prefix + '_' + @page_cnt.to_s + '.log'
|
|
74
|
+
@s3.bucket(command_options[:bucket]).object(object_name).put(
|
|
74
75
|
:body => page_content
|
|
75
76
|
)
|
|
76
77
|
Escort::Logger.output.puts "Put #{object_name} to S3."
|