fluent-plugin-elb-log 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b28cbf57d3635f360469e3aa85b5578fa22d42e6
+  data.tar.gz: 85f84e3f4af74a545c98149322a06cf8728f048e
+SHA512:
+  metadata.gz: b01dfd0527825704b8504f3162ae83091b316cfae1fff9426fa8e5be94e0d735a3da754d9b6a97a4cbc402e38caf176c7f2a4271284d3320952f8eec749790b4
+  data.tar.gz: f3c679a1f783b92dba3233de478216afcbf78fd730df3f8219e299b3c264deecd33fd73988a3ad6711aa05ccf074876f9299008a1652bd75efc70281ab22e464

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elb-log.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 shinsaka
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,71 @@
+# Amazon ELB log input plugin for fluentd
+
+## Overview
+- Amazon Web Services ELB log input plubin for fluentd
+
+## Installation
+
+    $ fluentd-gem fluent-plugin-elb-log
+
+## AWS ELB Settings
+- settings see: [Elastic Load Balancing](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html)
+- developer guide: [](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html)
+
+## Configuration
+
+```config
+<source>
+  type elb_log
+  access_key_id     <access_key>
+  secret_access_key <secret_access_key>
+  s3_endpoint       <s3_endpoint>
+  s3_bucketname     <bucketname>
+  s3_prefix         <elb log's prefix>
+  timestamp_file    <proc last file timestamp record filename>
+  refresh_interval  <interval number by second>
+</source>
+```
+
+### Example setting
+```config
+<source>
+  type elb_log
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  s3_endpoint       s3.amazonaws.com
+  s3_bucketname     bucketname
+  s3_prefix         prefix
+  timestamp_file    elb_last_at.dat
+  refresh_interval  300
+</source>
+
+<match **>
+  type stdout
+</match>
+```json
+{
+    "account_id": "999999999999", 
+    "backend": "192.168.30.127", 
+    "backend_port": "80", 
+    "backend_processing_time": "0.000985", 
+    "backend_status_code": "200", 
+    "client": "118.20.x.x", 
+    "client_port": "46171", 
+    "elb": "fluent-test-elb", 
+    "elb_ip_address": "54.250.x.x", 
+    "elb_status_code": "200", 
+    "logfile_date": "2014/03/09", 
+    "logfile_elb_name": "fluent-test-elb", 
+    "logfile_hash": "xyz123ab", 
+    "received_bytes": "0", 
+    "region": "ap-northeast-1", 
+    "request_method": "GET", 
+    "request_processing_time": "0.000072", 
+    "request_protocol": "HTTP/1.1", 
+    "request_uri": "http://logfile_elb_name-00000000.ap-northeast-1.elb.amazonaws.com:80/", 
+    "response_processing_time": "0.00007", 
+    "sent_bytes": "9", 
+    "time": "2014-03-09T04:10:33.785083Z"
+}
+```
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-elb-log.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-elb-log"
+  spec.version       = "0.0.2"
+  spec.authors       = ["shinsaka"]
+  spec.email         = ["shinx1265@gmail.com"]
+  spec.summary       = "Amazon ELB log input plugin"
+  spec.description   = "Amazon ELB log input plugin for fluentd"
+  spec.homepage      = "https://github.com/shinsaka/fluent-plugin-elb-log"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "aws-sdk"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

data/fluent.conf.sample

@@ -0,0 +1,15 @@
+<source>
+  type elb_log
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  s3_endpoint       s3.amazonaws.com
+  s3_bucketname     bucketname
+  s3_prefix         prefix
+  timestamp_file    elb_last_at.dat
+  refresh_interval  300
+</source>
+
+<match elb.access>
+  type stdout
+</match>
+

data/lib/fluent/plugin/in_elb_log.rb

@@ -0,0 +1,136 @@
+class Fluent::Elb_LogInput < Fluent::Input
+  Fluent::Plugin.register_input('elb_log', self)
+  
+  LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
+  ACCESSLOG_REGEXP = /^(?<time>.+?) (?<elb>.+?) (?<client>.+)\:(?<client_port>.+) (?<backend>.+)\:(?<backend_port>.+) (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"$/
+
+  config_param :access_key_id, :string, :default => nil
+  config_param :secret_access_key, :string, :default => nil
+  config_param :s3_bucketname, :string, :default => nil
+  config_param :s3_prefix, :string, :default => nil
+  config_param :s3_endpoint, :string, :default => 's3.amazon.com'
+  config_param :timestamp_file, :string, :default => nil
+  config_param :refresh_interval, :integer, :default => 300
+
+  def configure(conf)
+    super
+    require 'aws-sdk'
+    raise Fluent::ConfigError.new("access_key_id is required") unless @access_key_id
+    raise Fluent::ConfigError.new("secret_access_key is required") unless @secret_access_key
+    raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
+    raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
+  end
+
+  def start
+    super
+
+    init_s3bucket
+
+    @timestamp_file = File.open(@timestamp_file, File::RDWR|File::CREAT)
+    @timestamp_file.sync = true
+
+    @loop = Coolio::Loop.new
+    timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
+    timer_trigger.attach(@loop)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    super
+    @loop.stop
+    @thread.join
+    @timestamp_file.close
+  end
+
+  private
+
+  def init_s3bucket
+   @bucket = AWS::S3.new(
+      access_key_id: @access_key_id,
+      secret_access_key: @secret_access_key
+    ).buckets[@s3_bucketname]
+  end
+
+  def run
+    @loop.run
+  end
+
+  def input
+    $log.info "fluent-plugin-elb-log: input start"
+   
+    #get timestamp last proc
+    @timestamp_file.rewind
+    timestamp = @timestamp_file.read.to_i
+    timestamp = 0 unless timestamp
+    $log.info "fluent-plugin-elb-log: timestamp at start: " + Time.at(timestamp).to_s
+
+    log_objects = []
+    @bucket.objects.each do |obj|
+      next if obj.last_modified.to_i <= timestamp
+      matches = LOGFILE_REGEXP.match(obj.key)
+      next unless matches
+      next if !@s3_prefix.nil? && matches[:prefix] != @s3_prefix
+      log_objects.push obj
+    end
+
+    # sort by timestamp
+    log_objects.sort! do |a,b|
+      LOGFILE_REGEXP.match(a.key)[:elb_timestamp] <=> LOGFILE_REGEXP.match(b.key)[:elb_timestamp]
+    end
+
+    log_objects.each do |obj|
+      matches = LOGFILE_REGEXP.match(obj.key)
+      timestamp = matches[:elb_timestamp].to_i
+      record_common = {
+        account_id: matches[:account_id],
+        region: matches[:region],
+        logfile_date: matches[:logfile_date],
+        logfile_elb_name: matches[:logfile_elb_name],
+        elb_ip_address: matches[:elb_ip_address],
+        logfile_hash: matches[:logfile_hash],
+      }
+
+      obj.read do |line|
+        line_match = ACCESSLOG_REGEXP.match(line)
+        next unless line_match
+
+        record = {
+          time: line_match[:time],
+          elb: line_match[:elb],
+          client: line_match[:client],
+          client_port: line_match[:client_port],
+          backend: line_match[:backend],
+          backend_port: line_match[:backend_port],
+          request_processing_time: line_match[:request_processing_time],
+          backend_processing_time: line_match[:backend_processing_time],
+          response_processing_time: line_match[:response_processing_time],
+          elb_status_code: line_match[:elb_status_code],
+          backend_status_code: line_match[:backend_status_code],
+          received_bytes: line_match[:received_bytes],
+          sent_bytes: line_match[:sent_bytes],
+          request_method: line_match[:request_method],
+          request_uri: line_match[:request_uri],
+          request_protocol: line_match[:request_protocol],
+        }
+        
+        Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
+      end
+      # timestamp save
+      @timestamp_file.rewind
+      @timestamp_file.write(obj.last_modified.to_i)
+      @timestamp_file.truncate(@timestamp_file.tell)
+      $log.info "fluent-plugin-elb-log: timestamp save: " + obj.last_modified.to_s
+    end
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, &callback)
+      @callback = callback
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    end
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-elb-log
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- shinsaka
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Amazon ELB log input plugin for fluentd
+email:
+- shinx1265@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-elb-log.gemspec
+- fluent.conf.sample
+- lib/fluent/plugin/in_elb_log.rb
+homepage: https://github.com/shinsaka/fluent-plugin-elb-log
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Amazon ELB log input plugin
+test_files: []