fluent-plugin-cloudfront-log 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e41573e85cd6b33115e5f1525475c09c8668c099
-  data.tar.gz: ade5e4dd5a8ba5bdcc52de52396840ea4fd8ef0e
+  metadata.gz: 5d748be8ee5e67aa860a241c5f1397a91637bd62
+  data.tar.gz: 368fc0b0eed3682eab165301d2a6fae2be653716
 SHA512:
-  metadata.gz: 65fcc533f4fb58e998a60a2b6bbc9e9e48088b1b7c8d5268b25fd0aec426ed3a0e512100a7abe9255816c3e4b49544a0db015ef85e6ce36c9c36fcc7a7b8d43a
-  data.tar.gz: 353fb1744667a60477e2cb03b91d7007442766823922afeac18eca682b7bad2ac6fce969205adf7b0edaddaa5c52e554f4fea7e145e21fa2e12881e8dec50f91
+  metadata.gz: e6c81c25e0bbe1aaee20058e26caff7d5d697967143302323633b7da87f73599752574023411855050b60c3bdb3f4ee087b3594f23f04b3e6b23566076812ecf
+  data.tar.gz: 62f9f3e73c36c0e1b1f997274eaa0d4215ff7691a42cffce0d0c541754e282a3938ef383eb04ad1e7e3a3f43be427aa574d36ed5319a48b7d1b888f30b0753eb

data/README.md

@@ -1,8 +1,56 @@
 # Fluent::Plugin::Cloudfront::Log
+This plugin will connect to the S3 bucket that you store your cloudfront logs in. Once the plugin processes them and ships them to FluentD, it moves them to another location(either another bucket or sub directory).
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/cloudfront/log`. To experiment with that code, run `bin/console` for an interactive prompt.
+## Example config
+```
+<source>
+@type       cloudfront_log
+log_bucket  cloudfront-logs
+log_prefix  production
+region      us-east-1
+interval    300
+aws_key_id  xxxxxx
+aws_sec_key xxxxxx
+tag         reverb.cloudfront
+verbose     true
+</source>
+```
+
+## Configuration options
+
+#### log_bucket
+This option tells the plugin where to look for the cloudfront logs
+
+#### log_prefix
+For example if your logs are stored in a folder called "production" under the "cloudfront-logs" bucket, your logs would be stored in cloudfront like "cloudfront-logs/production/log.gz".
+In this case, you'd want to use the prefix "production".
+
+#### moved_log_bucket
+Here you can specify where you'd like the log files to be moved after processing. If left blank this defaults to a folder called `_moved` under the bucket configured for `@log_bucket`.
+
+#### moved_log_prefix
+This specifices what the log files will be named once they're processed. This defaults to `_moved`.
+
+#### region
+The region where your cloudfront logs are stored.
 
-TODO: Delete this and the text above, and describe your gem
+#### interval
+This is the rate in seconds at which we check the bucket for updated logs. It's recommended not to put this lower than 300(The default), cloudfront delivers logs every 20~ minutes to s3, so shortening this interval won't deliver your logs faster.
+
+#### aws_sec_id
+The ID of your AWS keypair. Note: Since this plugin uses aws-sdk under the hood you can leave these two aws fields blank if you have an IAM role applied to your FluentD instance.
+
+#### aws_sec_key
+The secret key portion of your AWS keypair
+
+#### tag
+This is a FluentD builtin.
+
+#### delimiter
+You shouldn't have to specify delimiter at all but this option is provided and passed to the S3 client in the event that you have a weird delimiter in your log file names. Defaults to `nil`.
+
+#### verbose
+Turn this on if you'd like to see verbose information about the plugin and how it's processing your files.
 
 ## Installation
 
@@ -14,15 +62,11 @@ gem 'fluent-plugin-cloudfront-log'
 
 And then execute:
 
-    $ bundle
+$ bundle
 
 Or install it yourself as:
 
-    $ gem install fluent-plugin-cloudfront-log
-
-## Usage
-
-TODO: Write usage instructions here
+$ gem install fluent-plugin-cloudfront-log
 
 ## Development
 

data/fluent-plugin-cloudfront-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cloudfront-log"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["kubihie"]
   spec.email         = ["kubihie@gmail.com"]
 

data/lib/fluent/plugin/in_cloudfront_log.rb

@@ -4,12 +4,14 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
   config_param :aws_key_id,        :string,  :default => nil, :secret => true
   config_param :aws_sec_key,       :string,  :default => nil, :secret => true
   config_param :log_bucket,        :string
-  config_param :log_prefix,        :string,  :default => nil
+  config_param :log_prefix,        :string
   config_param :moved_log_bucket,  :string,  :default => @log_bucket
   config_param :moved_log_prefix,  :string,  :default => '_moved'
   config_param :region,            :string
   config_param :tag,               :string,  :default => 'cloudfront.access'
-  config_param :interval,          :integer,    :default => 300
+  config_param :interval,          :integer, :default => 300
+  config_param :delimiter,         :string,  :default => nil
+  config_param :verbose,           :string,  :default => false
 
   def initialize
     super
@@ -25,24 +27,25 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
 
     raise Fluent::ConfigError.new unless @log_bucket
     raise Fluent::ConfigError.new unless @region
+    raise Fluent::ConfigError.new unless @log_prefix
 
     @moved_log_bucket = @log_bucket unless @moved_log_bucket
     @moved_log_prefix = @log_prefix + '_moved' unless @moved_log_prefix
-    
-    log.info('@log_bucket:' + @log_bucket)
-    log.info('@moved_log_bucket:' + @moved_log_bucket)
-    log.info('@log_prefix:' + @log_prefix)
-    log.info('@moved_log_prefix:' + @moved_log_prefix)
+
+    if @verbose
+      log.info("@log_bucket: #{@log_bucket}")
+      log.info("@moved_log_bucket: #{@moved_log_bucket}")
+      log.info("@log_prefix: #{@log_prefix}")
+      log.info("@moved_log_prefix: #{@moved_log_prefix}")
+    end
   end
 
   def start
     super
+    log.info("Cloudfront verbose logging enabled") if @verbose
     client
 
     @loop = Coolio::Loop.new
-    #timer = TimerWatcher.new(@interval, true, log) do
-    #  input
-    #end
     timer = TimerWatcher.new(@interval, true, log, &method(:input)) 
 
     @loop.attach(timer)
@@ -81,17 +84,27 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
   end
 
   def purge(filename)
-    log.info('copy_src:' + @log_bucket + '/' + @log_prefix + filename)
-    log.info('copy_dst:' + @moved_log_bucket + '/' + @moved_log_prefix + '/' + filename)
-    client.copy_object(:bucket => @moved_log_bucket, :copy_source => @log_bucket + '/' + @log_prefix + filename, :key => @moved_log_prefix + '/' + filename)
-    client.delete_object(:bucket => @log_bucket, :key => @log_prefix + filename)
+    # Key is the name of the object without the bucket prefix, e.g: asdf/asdf.jpg
+    source_object_key       = [@log_prefix, filename].join('/')
+
+    # Full path includes bucket name in addition to object key, e.g: bucket/asdf/asdf.jpg
+    source_object_full_path = [@log_bucket, source_object_key].join('/')
+
+    dest_object_key         = [@moved_log_prefix, filename].join('/')
+    dest_object_full_path   = [@moved_log_bucket, dest_object_key].join('/')
+
+    log.info("Copying object: #{source_object_full_path} to #{dest_object_full_path}") if @verbose
+    client.copy_object(:bucket => @moved_log_bucket, :copy_source => source_object_full_path, :key => dest_object_key)
+
+    log.info("Deleting object: #{source_object_key} from #{@log_bucket}") if @verbose
+    client.delete_object(:bucket => @log_bucket, :key => source_object_key)
   end
 
   def input
-    client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => '/').each do |list|
+    client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => @delimiter).each do |list|
       list.contents.each do |content|
-        filename = content.key.sub(/#{@log_prefix}/, '')
-        #log.info('filename:' + filename)
+        filename = content.key.sub(/^#{@log_prefix}\//, "")
+        log.info("Currently processing: #{filename}") if @verbose
         next if filename[-1] == '/'  #skip directory/
         next unless filename[-2, 2] == 'gz'  #skip without gz file
 
@@ -107,7 +120,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
           line = URI.unescape(line)  #hoge%20fuga   -> hoge fuga
           line = line.split("\t")
           record = Hash[@fields.collect.zip(line)]
-          timestamp = Time.parse("#{record['date']} #{record['time']} +0900").to_i
+          timestamp = Time.parse("#{record['date']}T#{record['time']}+00:00").to_i
           router.emit(@tag, timestamp, record)
         end
         purge(filename)

data/test/plugin/test_in_cloudfrontlog.rb

@@ -15,7 +15,9 @@ class Cloudfront_LogInputTest < Test::Unit::TestCase
     :moved_log_prefix  => 'a/b/c_moved',
     :region            => 'ap-northeast-1',
     :tag               => 'cloudfront',
-    :interval          => '500'
+    :interval          => '500',
+    :delimiter         => nil,
+    :verbose           => true,
   }
 
   def parse_config(conf = {})
@@ -43,6 +45,13 @@ class Cloudfront_LogInputTest < Test::Unit::TestCase
     }
     assert_equal("'region' parameter is required", exception.message)
 
+    exception = assert_raise(Fluent::ConfigError) {
+      conf = DEFAULT_CONFIG.clone
+      conf.delete(:log_prefix)
+      driver = create_driver(conf)
+    }
+    assert_equal("'log_prefix' parameter is required", exception.message)
+
     conf = DEFAULT_CONFIG.clone
     conf.delete(:moved_log_bucket)
     driver = create_driver(conf)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudfront-log
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - kubihie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-01 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -114,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.4.7
 signing_key: 
 specification_version: 4
 summary: AWS CloudFront log input plugin.