cwlogs-s3 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +14 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +33 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +2 -0
- data/bin/cwlogs-s3 +28 -0
- data/cwlogs-s3.gemspec +22 -0
- data/lib/cwlogs-s3/command.rb +81 -0
- data/lib/cwlogs-s3.rb +1 -0
- metadata +89 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
aws-sdk (2.0.27)
|
5
|
+
aws-sdk-resources (= 2.0.27)
|
6
|
+
aws-sdk-core (2.0.27)
|
7
|
+
builder (~> 3.0)
|
8
|
+
jmespath (~> 1.0)
|
9
|
+
multi_json (~> 1.0)
|
10
|
+
multi_xml (~> 0.5)
|
11
|
+
aws-sdk-resources (2.0.27)
|
12
|
+
aws-sdk-core (= 2.0.27)
|
13
|
+
builder (3.2.2)
|
14
|
+
chronic (0.10.2)
|
15
|
+
chronic_duration (0.10.6)
|
16
|
+
numerizer (~> 0.1.1)
|
17
|
+
escort (0.4.0)
|
18
|
+
nesty
|
19
|
+
jmespath (1.0.2)
|
20
|
+
multi_json (~> 1.0)
|
21
|
+
multi_json (1.10.1)
|
22
|
+
multi_xml (0.5.5)
|
23
|
+
nesty (1.0.2)
|
24
|
+
numerizer (0.1.1)
|
25
|
+
|
26
|
+
PLATFORMS
|
27
|
+
ruby
|
28
|
+
|
29
|
+
DEPENDENCIES
|
30
|
+
aws-sdk (~> 2.0.27)
|
31
|
+
chronic (~> 0.10.2)
|
32
|
+
chronic_duration (~> 0.10.6)
|
33
|
+
escort (~> 0.4.0)
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Tim-B
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# Cwlogs::S3
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'cwlogs-s3'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install cwlogs-s3
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
TODO: Write usage instructions here
|
24
|
+
|
25
|
+
## Contributing
|
26
|
+
|
27
|
+
1. Fork it ( https://github.com/[my-github-username]/cwlogs-s3/fork )
|
28
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
29
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
30
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
31
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
data/bin/cwlogs-s3
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'escort'
|
4
|
+
require 'chronic'
|
5
|
+
require 'chronic_duration'
|
6
|
+
|
7
|
+
require 'cwlogs-s3'
|
8
|
+
|
9
|
+
Escort::App.create do |app|
|
10
|
+
|
11
|
+
app.options do |opts|
|
12
|
+
opts.opt :group, 'Log group name', :short => '-g', :long => '--group', :type => :string, default: nil
|
13
|
+
opts.opt :period, 'Period to export', :short => '-p', :long => '--period', :type => :string, default: '1 day'
|
14
|
+
opts.opt :ending, 'Time when period ends', :short => '-e', :long => '--ending', :type => :string, default: 'now'
|
15
|
+
opts.opt :bucket, 'Destination bucket', :short => '-b', :long => '--bucket', :type => :string, default: nil
|
16
|
+
opts.opt :prefix, 'Prefix', :short => '-f', :long => '--prefix', :type => :string, default: 'logs/'
|
17
|
+
opts.opt :region, 'AWS region', :short => '-r', :long => '--region', :type => :string, default: 'us-east-1'
|
18
|
+
|
19
|
+
opts.validate(:period, 'Cannot parse period') { |option| ChronicDuration.parse(option) != nil }
|
20
|
+
opts.validate(:ending, 'Cannot parse ending') { |option| Chronic.parse(option) != nil }
|
21
|
+
opts.validate(:group, 'Log group is required') { |option| option != nil }
|
22
|
+
opts.validate(:bucket, 'Bucket is required') { |option| option != nil }
|
23
|
+
end
|
24
|
+
|
25
|
+
app.action do |options, arguments|
|
26
|
+
CWLogsToS3::Command.new(options, arguments).execute
|
27
|
+
end
|
28
|
+
end
|
data/cwlogs-s3.gemspec
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "cwlogs-s3"
|
7
|
+
spec.version = '0.0.1'
|
8
|
+
spec.authors = ["Tim-B"]
|
9
|
+
spec.email = ["tim@galacticcode.com"]
|
10
|
+
spec.summary = %q{Exports logs from CloudWatch logs and uploads them to S3}
|
11
|
+
spec.description = %q{Exports logs from CloudWatch logs and uploads them to S3}
|
12
|
+
spec.homepage = ""
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
21
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
22
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'escort'
|
2
|
+
require 'chronic'
|
3
|
+
require 'chronic_duration'
|
4
|
+
require 'aws-sdk'
|
5
|
+
require 'securerandom'
|
6
|
+
|
7
|
+
module CWLogsToS3
|
8
|
+
|
9
|
+
class Command < ::Escort::ActionCommand::Base
|
10
|
+
|
11
|
+
def execute
|
12
|
+
|
13
|
+
@event_cnt = 0
|
14
|
+
@page_cnt = 0
|
15
|
+
|
16
|
+
ending = Chronic.parse(command_options[:ending])
|
17
|
+
period = ChronicDuration.parse(command_options[:period])
|
18
|
+
start = Time.at(ending.to_i - period)
|
19
|
+
|
20
|
+
Escort::Logger.output.puts "Exporting from #{start} to #{ending}"
|
21
|
+
|
22
|
+
@cwl = Aws::CloudWatchLogs::Client.new(region: command_options[:region])
|
23
|
+
@s3 = Aws::S3::Resource.new(region: command_options[:region])
|
24
|
+
|
25
|
+
@stream_list = []
|
26
|
+
|
27
|
+
resp = @cwl.describe_log_streams(
|
28
|
+
log_group_name: command_options[:group],
|
29
|
+
)
|
30
|
+
|
31
|
+
resp.each do |streams|
|
32
|
+
streams[:log_streams].each do |stream|
|
33
|
+
@stream_list.push stream[:log_stream_name]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
Escort::Logger.output.puts "Exporting streams #{@stream_list.join(', ')}"
|
38
|
+
|
39
|
+
page = 0
|
40
|
+
|
41
|
+
@stream_list.each do |stream|
|
42
|
+
Escort::Logger.output.puts "Starting stream #{stream}"
|
43
|
+
resp = @cwl.get_log_events(
|
44
|
+
log_group_name: command_options[:group],
|
45
|
+
log_stream_name: stream,
|
46
|
+
start_time: start.to_i * 1000,
|
47
|
+
end_time: ending.to_i * 1000,
|
48
|
+
)
|
49
|
+
|
50
|
+
resp.each do |log_page|
|
51
|
+
if log_page[:events].length == 0
|
52
|
+
break
|
53
|
+
end
|
54
|
+
put_page log_page[:events]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
Escort::Logger.output.puts "Finished, #{@event_cnt} events extracted."
|
59
|
+
end
|
60
|
+
|
61
|
+
def put_page events
|
62
|
+
page_content = ''
|
63
|
+
events.each do |event|
|
64
|
+
@event_cnt = @event_cnt + 1
|
65
|
+
page_content << event[:message] << "\n"
|
66
|
+
end
|
67
|
+
object_name = command_options[:prefix] + randomise_prefix + '_' + @page_cnt.to_s + '.log'
|
68
|
+
@s3.bucket(command_options[:bucket]).object(object_name).put(
|
69
|
+
:body => page_content
|
70
|
+
)
|
71
|
+
Escort::Logger.output.puts "Put #{object_name} to S3."
|
72
|
+
@page_cnt = @page_cnt + 1
|
73
|
+
end
|
74
|
+
|
75
|
+
def randomise_prefix
|
76
|
+
SecureRandom.hex(2).to_s
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
data/lib/cwlogs-s3.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/cwlogs-s3/command'
|
metadata
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cwlogs-s3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Tim-B
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-02-22 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.7'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.7'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '10.0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '10.0'
|
46
|
+
description: Exports logs from CloudWatch logs and uploads them to S3
|
47
|
+
email:
|
48
|
+
- tim@galacticcode.com
|
49
|
+
executables:
|
50
|
+
- cwlogs-s3
|
51
|
+
extensions: []
|
52
|
+
extra_rdoc_files: []
|
53
|
+
files:
|
54
|
+
- .gitignore
|
55
|
+
- Gemfile
|
56
|
+
- Gemfile.lock
|
57
|
+
- LICENSE.txt
|
58
|
+
- README.md
|
59
|
+
- Rakefile
|
60
|
+
- bin/cwlogs-s3
|
61
|
+
- cwlogs-s3.gemspec
|
62
|
+
- lib/cwlogs-s3.rb
|
63
|
+
- lib/cwlogs-s3/command.rb
|
64
|
+
homepage: ''
|
65
|
+
licenses:
|
66
|
+
- MIT
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ! '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
requirements: []
|
84
|
+
rubyforge_project:
|
85
|
+
rubygems_version: 1.8.23
|
86
|
+
signing_key:
|
87
|
+
specification_version: 3
|
88
|
+
summary: Exports logs from CloudWatch logs and uploads them to S3
|
89
|
+
test_files: []
|