cwlogs-s3 0.0.3 → 0.0.4

data/README.md

@@ -1,30 +1,54 @@
-# Cwlogs::S3
+# cwlogs-s3
 
-TODO: Write a gem description
+Task for exporting CloudWatch logs to S3. Useful for then running logs through EMR for analysis. Designed to be run
+ via AWS Data Pipeline.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+    $ gem install cwlogs-s3
+
+## Usage
 
-```ruby
-gem 'cwlogs-s3'
+```
+NAME
+    cwlogs-s3 -
+
+USAGE
+    cwlogs-s3 [options] [arguments]
+
+OPTIONS
+    --group -g <s>           - Log group name
+                               - Log group is required
+    --period -p <s>          - Period to export (default: 1 day)
+                               - Cannot parse period
+    --ending -e <s>          - Time when period ends (default: now)
+                               - Cannot parse ending
+    --s3path -s <s>          - Destination S3 path
+                               - S3 path is required
+    --region -r <s>          - AWS region (default: us-east-1)
+    --verbosity <s>          - Verbosity level of output for current execution
+                               (e.g. INFO, DEBUG) (default: WARN)
+    --error-output-format <s - The format to use when outputting errors (e.g. b
+    >                          asic, advanced) (default: basic)
+    --help -h                - Show this messag
 ```
 
-And then execute:
+For example:
 
-    $ bundle
+```
+cwlogs-s3 -g "my-app-http-access" -p "3 days" -e "now" -s "s3://mybucket/mypath/"
+```
 
-Or install it yourself as:
+Which will export all logs in the `my-app-http-access` from the last 3 days ending now to a bucket called `my-log-bucket`.
 
-    $ gem install cwlogs-s3
 
-## Usage
+`--period` Can be formatted according to Chronic Duration: https://github.com/hpoydar/chronic_duration
 
-TODO: Write usage instructions here
+`--ending` Can be formatted according to Chronic: https://github.com/mojombo/chronic
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/cwlogs-s3/fork )
+1. Fork it ( https://github.com/Tim-B/cwlogs-s3/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/bin/cwlogs-s3

@@ -12,14 +12,13 @@ Escort::App.create do |app|
     opts.opt :group, 'Log group name', :short => '-g', :long => '--group', :type => :string, default: nil
     opts.opt :period, 'Period to export', :short => '-p', :long => '--period', :type => :string, default: '1 day'
     opts.opt :ending, 'Time when period ends', :short => '-e', :long => '--ending', :type => :string, default: 'now'
-    opts.opt :bucket, 'Destination bucket', :short => '-b', :long => '--bucket', :type => :string, default: nil
-    opts.opt :prefix, 'Prefix', :short => '-f', :long => '--prefix', :type => :string, default: 'logs/'
+    opts.opt :s3_path, 'Destination S3 path', :short => '-s', :long => '--s3path', :type => :string, default: nil
     opts.opt :region, 'AWS region', :short => '-r', :long => '--region', :type => :string, default: 'us-east-1'
 
     opts.validate(:period, 'Cannot parse period') { |option| ChronicDuration.parse(option) != nil }
     opts.validate(:ending, 'Cannot parse ending') { |option| Chronic.parse(option) != nil }
     opts.validate(:group, 'Log group is required') { |option| option != nil }
-    opts.validate(:bucket, 'Bucket is required') { |option| option != nil }
+    opts.validate(:s3_path, 'S3 path is required') { |option| option != nil }
   end
 
   app.action do |options, arguments|

data/cwlogs-s3.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "cwlogs-s3"
-  spec.version       = '0.0.3'
+  spec.version       = '0.0.4'
   spec.authors       = ["Tim-B"]
   spec.email         = ["tim@galacticcode.com"]
   spec.summary       = %q{Exports logs from CloudWatch logs and uploads them to S3}

data/lib/cwlogs-s3/command.rb

@@ -3,6 +3,7 @@ require 'chronic'
 require 'chronic_duration'
 require 'aws-sdk'
 require 'securerandom'
+require 'uri'
 
 module CWLogsToS3
 
@@ -17,6 +18,10 @@ module CWLogsToS3
       period = ChronicDuration.parse(command_options[:period])
       start = Time.at(ending.to_i - period)
 
+      s3Uri = URI(command_options[:s3_path])
+      @bucket = s3Uri.host
+      @s3_prefix = s3Uri.path.gsub!(/^\//, '')
+
       Escort::Logger.output.puts "Exporting from #{start} to #{ending}"
 
       @cwl = Aws::CloudWatchLogs::Client.new(region: command_options[:region])
@@ -64,8 +69,8 @@ module CWLogsToS3
         @event_cnt = @event_cnt + 1
         page_content << event[:message] << "\n"
       end
-      object_name = command_options[:prefix] + randomise_prefix + '_' + @page_cnt.to_s + '.log'
-      @s3.bucket(command_options[:bucket]).object(object_name).put(
+      object_name = @s3_prefix + randomise_prefix + '_' + @page_cnt.to_s + '.log'
+      @s3.bucket(@bucket).object(object_name).put(
         :body => page_content
       )
       Escort::Logger.output.puts "Put #{object_name} to S3."

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cwlogs-s3
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-22 00:00:00.000000000 Z
+date: 2015-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler