cwlogs-s3 0.0.1

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem 'escort', '~> 0.4.0'
+gem 'aws-sdk', '~> 2.0.27'
+gem 'chronic_duration', '~> 0.10.6'
+gem 'chronic', '~> 0.10.2'

data/Gemfile.lock

@@ -0,0 +1,33 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.0.27)
+      aws-sdk-resources (= 2.0.27)
+    aws-sdk-core (2.0.27)
+      builder (~> 3.0)
+      jmespath (~> 1.0)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+    aws-sdk-resources (2.0.27)
+      aws-sdk-core (= 2.0.27)
+    builder (3.2.2)
+    chronic (0.10.2)
+    chronic_duration (0.10.6)
+      numerizer (~> 0.1.1)
+    escort (0.4.0)
+      nesty
+    jmespath (1.0.2)
+      multi_json (~> 1.0)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    nesty (1.0.2)
+    numerizer (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk (~> 2.0.27)
+  chronic (~> 0.10.2)
+  chronic_duration (~> 0.10.6)
+  escort (~> 0.4.0)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Tim-B
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# Cwlogs::S3
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'cwlogs-s3'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cwlogs-s3
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/cwlogs-s3/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/cwlogs-s3

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'escort'
+require 'chronic'
+require 'chronic_duration'
+
+require 'cwlogs-s3'
+
+Escort::App.create do |app|
+
+  app.options do |opts|
+    opts.opt :group, 'Log group name', :short => '-g', :long => '--group', :type => :string, default: nil
+    opts.opt :period, 'Period to export', :short => '-p', :long => '--period', :type => :string, default: '1 day'
+    opts.opt :ending, 'Time when period ends', :short => '-e', :long => '--ending', :type => :string, default: 'now'
+    opts.opt :bucket, 'Destination bucket', :short => '-b', :long => '--bucket', :type => :string, default: nil
+    opts.opt :prefix, 'Prefix', :short => '-f', :long => '--prefix', :type => :string, default: 'logs/'
+    opts.opt :region, 'AWS region', :short => '-r', :long => '--region', :type => :string, default: 'us-east-1'
+
+    opts.validate(:period, 'Cannot parse period') { |option| ChronicDuration.parse(option) != nil }
+    opts.validate(:ending, 'Cannot parse ending') { |option| Chronic.parse(option) != nil }
+    opts.validate(:group, 'Log group is required') { |option| option != nil }
+    opts.validate(:bucket, 'Bucket is required') { |option| option != nil }
+  end
+
+  app.action do |options, arguments|
+    CWLogsToS3::Command.new(options, arguments).execute
+  end
+end

data/cwlogs-s3.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "cwlogs-s3"
+  spec.version       = '0.0.1'
+  spec.authors       = ["Tim-B"]
+  spec.email         = ["tim@galacticcode.com"]
+  spec.summary       = %q{Exports logs from CloudWatch logs and uploads them to S3}
+  spec.description   = %q{Exports logs from CloudWatch logs and uploads them to S3}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/cwlogs-s3/command.rb

@@ -0,0 +1,81 @@
+require 'escort'
+require 'chronic'
+require 'chronic_duration'
+require 'aws-sdk'
+require 'securerandom'
+
+module CWLogsToS3
+
+  class Command < ::Escort::ActionCommand::Base
+
+    def execute
+
+      @event_cnt = 0
+      @page_cnt = 0
+
+      ending = Chronic.parse(command_options[:ending])
+      period = ChronicDuration.parse(command_options[:period])
+      start = Time.at(ending.to_i - period)
+
+      Escort::Logger.output.puts "Exporting from #{start} to #{ending}"
+
+      @cwl = Aws::CloudWatchLogs::Client.new(region: command_options[:region])
+      @s3 = Aws::S3::Resource.new(region: command_options[:region])
+
+      @stream_list = []
+
+      resp = @cwl.describe_log_streams(
+          log_group_name: command_options[:group],
+      )
+
+      resp.each do |streams|
+        streams[:log_streams].each do |stream|
+          @stream_list.push stream[:log_stream_name]
+        end
+      end
+
+      Escort::Logger.output.puts "Exporting streams #{@stream_list.join(', ')}"
+
+      page = 0
+
+      @stream_list.each do |stream|
+        Escort::Logger.output.puts "Starting stream #{stream}"
+        resp = @cwl.get_log_events(
+            log_group_name: command_options[:group],
+            log_stream_name: stream,
+            start_time: start.to_i * 1000,
+            end_time: ending.to_i * 1000,
+        )
+
+        resp.each do |log_page|
+          if log_page[:events].length == 0
+            break
+          end
+          put_page log_page[:events]
+        end
+      end
+
+      Escort::Logger.output.puts "Finished, #{@event_cnt} events extracted."
+    end
+
+    def put_page events
+      page_content = ''
+      events.each do |event|
+        @event_cnt = @event_cnt + 1
+        page_content << event[:message] << "\n"
+      end
+      object_name = command_options[:prefix] + randomise_prefix + '_' + @page_cnt.to_s + '.log'
+      @s3.bucket(command_options[:bucket]).object(object_name).put(
+        :body => page_content
+      )
+      Escort::Logger.output.puts "Put #{object_name} to S3."
+      @page_cnt = @page_cnt + 1
+    end
+
+    def randomise_prefix
+      SecureRandom.hex(2).to_s
+    end
+
+  end
+
+end

data/lib/cwlogs-s3.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/cwlogs-s3/command'

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: cwlogs-s3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Tim-B
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Exports logs from CloudWatch logs and uploads them to S3
+email:
+- tim@galacticcode.com
+executables:
+- cwlogs-s3
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/cwlogs-s3
+- cwlogs-s3.gemspec
+- lib/cwlogs-s3.rb
+- lib/cwlogs-s3/command.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Exports logs from CloudWatch logs and uploads them to S3
+test_files: []