fluent-plugin-elb-log 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc7ef89a1cfd7bb7770068aee0293be8b5c41aa9
-  data.tar.gz: 7f8b917cf9857f9c344a6fadee95d84a9846b086
+  metadata.gz: eb31f022a24f4237678c1666bbe9d407eaa858b3
+  data.tar.gz: 6c353760ad26a95b9bfc9293548750a8fe842058
 SHA512:
-  metadata.gz: a78ad895495865c24abe9a6b98d24e1e80bb72df191d69130db9992bcf84e0bea1b54890ac491850da0f8c993677f34f64b135ffde4768a5fc4417edb74fe403
-  data.tar.gz: 11ff9840bf03537893dd74ac2bcfd27c3520dd484ad7afc4853b5f54fcd8bcd85b2f5815531130ef80557137992deff12b2073d44a53e132a451b904767765a6
+  metadata.gz: 32102c4c6e34209171d5fd95f474d07bd227308bf85a7c4b67f1eeeb6e3822fa6824136271a771d5aa6933fb1d9080b1d46ba7c648cc536e3308acce67462b63
+  data.tar.gz: fe24ac63617e936272d650a55b1746101c31eebfd77de93f20f4cfccfa569697baf5f3065b83e73d80721f9fa9648052d11f55e413f78eadf5fa31346c2773ea

data/README.md

@@ -23,6 +23,7 @@
   s3_prefix         <elb log's prefix>
   timestamp_file    <proc last file timestamp record filename>
   refresh_interval  <interval number by second>
+  buf_file          <buffer file path>
 
   # following attibutes are required if you don't use IAM Role
   access_key_id     <access_key>
@@ -41,6 +42,7 @@
   s3_prefix         prefix
   timestamp_file    elb_last_at.dat
   refresh_interval  300
+  buf_file          /tmp/fluentd-elblog.tmpfile
 </source>
 
 <match **>

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.0.8"
+  spec.version       = "0.1.0"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"

data/lib/fluent/plugin/in_elb_log.rb

@@ -11,6 +11,7 @@ class Fluent::Elb_LogInput < Fluent::Input
   config_param :s3_endpoint, :string, :default => 's3.amazon.com'
   config_param :timestamp_file, :string, :default => nil
   config_param :refresh_interval, :integer, :default => 300
+  config_param :buf_file, :string, :default => './fluentd_elb_log_buf_file'
 
   def configure(conf)
     super
@@ -25,13 +26,13 @@ class Fluent::Elb_LogInput < Fluent::Input
 
     raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
     raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
+    raise Fluent::ConfigError.new("s3 bucket fetch error #{@s3_bucketname}") if init_s3bucket.nil?
+    FileUtils.touch(@buf_file)
   end
 
   def start
     super
 
-    init_s3bucket
-
     @timestamp_file = File.open(@timestamp_file, File::RDWR|File::CREAT)
     @timestamp_file.sync = true
 
@@ -58,7 +59,13 @@ class Fluent::Elb_LogInput < Fluent::Input
     end
     options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
 
-    @bucket = AWS::S3.new(options).buckets[@s3_bucketname]
+    begin
+      @bucket = AWS::S3.new(options).buckets[@s3_bucketname]
+      @bucket.objects.count
+    rescue => e
+      $log.warn "fluent-plugin-elb-log: s3 bucket fetch error: #{e.message}"
+      nil
+    end
   end
 
   def run
@@ -101,7 +108,27 @@ class Fluent::Elb_LogInput < Fluent::Input
         elb_timestamp: matches[:elb_timestamp],
       }
 
-      obj.read do |line|
+      # read an object from S3 to a file and write buffer file
+      File.open(@buf_file, File::WRONLY) do |file|
+        obj.read do |chunk|
+          file.write(chunk)
+        end
+      end
+
+      emit_lines_from_buffer_file record_common
+
+      # timestamp save
+      @timestamp_file.rewind
+      @timestamp_file.write(obj.last_modified.to_i)
+      @timestamp_file.truncate(@timestamp_file.tell)
+      $log.info "fluent-plugin-elb-log: timestamp save: " + obj.last_modified.to_s
+    end
+  end
+
+  def emit_lines_from_buffer_file(record_common)
+    # emit per line
+    File.open(@buf_file, File::RDONLY) do |file|
+      file.each_line do |line|
         line_match = ACCESSLOG_REGEXP.match(line)
         next unless line_match
 
@@ -126,11 +153,6 @@ class Fluent::Elb_LogInput < Fluent::Input
 
         Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
       end
-      # timestamp save
-      @timestamp_file.rewind
-      @timestamp_file.write(obj.last_modified.to_i)
-      @timestamp_file.truncate(@timestamp_file.tell)
-      $log.info "fluent-plugin-elb-log: timestamp save: " + obj.last_modified.to_s
     end
   end
 

data/test/plugin/in_elb_log.rb

@@ -98,16 +98,5 @@ class Elb_LogInputTest < Test::Unit::TestCase
     }
     assert_equal('secret_access_key is required', exception.message)
   end
-end
 
-#class Elb_LogInputFormatTest < Test::Unit::TestCase
-#
-#  def setup
-#    Fluent::Test.setup
-#  end
-#
-#  def test_format
-#    
-#  end
-#
-#end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-07 00:00:00.000000000 Z
+date: 2014-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd