fluent-plugin-s3 0.2.0 → 0.2.1

data/ChangeLog

@@ -1,4 +1,11 @@
 
+Release 0.2.1 - 2011/10/24
+
+  * Add sequential number to the file to avoid overwriting
+  * Use bundler instead of jeweler for packaging
+  * Updated README
+
+
 Release 0.2.0 - 2011/10/16
 
   * Updated to fluentd-0.10.0

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -1,2 +1,55 @@
 = Amazon S3 output plugin for Fluent event collector
 
+== Overview
+
+*s3* output plugin buffers event logs in local file and upload it to S3 periodically.
+
+This plugin splits files exactly by using the time of event logs (not the time when the logs are received). For example, a log '2011-01-02 message B' is reached, and then another log '2011-01-03 message B' is reached in this order, the former one is stored in "20110102.gz" file, and latter one in "20110103.gz" file.
+
+
+== Installation
+
+Simply use RubyGems:
+
+    gem install fluent-plugin-s3
+
+== Configuration
+
+    <match pattern>
+      type s3
+
+      aws_key_id YOUR_AWS_KEY_ID
+      aws_sec_key YOUR_AWS_SECRET/KEY
+      s3_bucket YOUR_S3_BUCKET_NAME
+      path logs/
+      buffer_path /var/log/fluent/s3
+
+      time_slice_format %Y%m%d-%H
+      time_slice_wait 10m
+      utc
+    </match>
+
+[aws_key_id (required)] AWS access key id.
+
+[aws_sec_key (required)] AWS secret key.
+
+[s3_bucket (required)] S3 bucket name.
+
+[path] path prefix of the files on S3. Default is "" (no prefix).
+
+[buffer_path (required)] path prefix of the files to buffer logs.
+
+[time_slice_format] Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
+
+[time_slice_wait] The time to wait old logs. Default is 10 minutes. Specify larger value if old logs may reache.
+
+[utc] Use UTC instead of local time.
+
+The actual path on S3 will be: "{path}{time_slice_format}_{sequential_number}.gz"
+
+
+== Copyright
+
+Copyright:: Copyright (c) 2011 Sadayuki Furuhashi
+License::   Apache License, Version 2.0
+

data/Rakefile

@@ -1,60 +1,14 @@
-require 'rake'
-require 'rake/testtask'
-require 'rake/clean'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "fluent-plugin-s3"
-    gemspec.summary = "Amazon S3 output plugin for Fluent event collector"
-    gemspec.author = "Sadayuki Furuhashi"
-    gemspec.email = "frsyuki@gmail.com"
-    gemspec.homepage = "http://fluent.github.com/"
-    gemspec.has_rdoc = false
-    gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluentd", "~> 0.10.0"
-    gemspec.add_dependency "aws-sdk", "~> 1.1.3"
-    gemspec.test_files = Dir["test/**/*.rb"]
-    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
-      %w[VERSION AUTHORS Rakefile]
-    gemspec.executables = []
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available. Install it with: gem install jeweler"
-end
-
-Rake::TestTask.new(:test) do |t|
-  t.test_files = Dir['test/*_test.rb']
-  t.ruby_opts = ['-rubygems'] if defined? Gem
-  t.ruby_opts << '-I.'
-end
 
-#VERSION_FILE = "lib/fluent/version.rb"
-#
-#file VERSION_FILE => ["VERSION"] do |t|
-#  version = File.read("VERSION").strip
-#  File.open(VERSION_FILE, "w") {|f|
-#    f.write <<EOF
-#module Fluent
-#
-#VERSION = '#{version}'
-#
-#end
-#EOF
-#  }
-#end
-#
-#task :default => [VERSION_FILE, :build]
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-task :default => [:build]
+require 'rake/testtask'
 
-Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.test_files = Dir["test/*.rb"].sort
-  t.verbose = true
-  #t.warning = true
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
 end
 
-task :default => [:build, :gemspec]
+task :default => [:build]
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

data/fluent-plugin-s3.gemspec

@@ -0,0 +1,21 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-s3"
+  gem.description = "Amazon S3 output plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/fluent/fluent-plugin-s3"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "aws-sdk", "~> 1.1.3"
+end

data/lib/fluent/plugin/out_s3.rb

@@ -9,50 +9,20 @@ class S3Output < Fluent::TimeSlicedOutput
     require 'aws-sdk'
     require 'zlib'
     require 'time'
+    require 'tempfile'
   end
 
-  attr_accessor :aws_key_id, :aws_sec_key, :s3_bucket, :path
+  config_param :path, :string, :default => ""
+  config_param :time_format, :string, :default => nil
+
+  config_param :aws_key_id, :string
+  config_param :aws_sec_key, :string
+  config_param :s3_bucket, :string
 
   def configure(conf)
     super
 
-    if aws_key_id = conf['aws_key_id']
-      @aws_key_id = aws_key_id
-    end
-    unless @aws_key_id
-      raise ConfigError, "'aws_key_id' parameter is required on s3 output"
-    end
-
-    if aws_sec_key = conf['aws_sec_key']
-      @aws_sec_key = aws_sec_key
-    end
-    unless @aws_sec_key
-      raise ConfigError, "'aws_sec_key' parameter is required on s3 output"
-    end
-
-    if s3_bucket = conf['s3_bucket']
-      @s3_bucket = s3_bucket
-    end
-    unless @s3_bucket
-      raise ConfigError, "'s3_bucket' parameter is required on s3 output"
-    end
-
-    if path = conf['path']
-      @path = path
-    end
-    unless @path
-      @path = ''
-    end
-
-    if @localtime
-      @formatter = Proc.new {|tag,time,record|
-        "#{Time.at(time).iso8601}\t#{tag}\t#{record.to_json}\n"
-      }
-    else
-      @formatter = Proc.new {|tag,time,record|
-        "#{Time.at(time).utc.iso8601}\t#{tag}\t#{record.to_json}\n"
-      }
-    end
+    @timef = TimeFormatter.new(@time_format, @localtime)
   end
 
   def start
@@ -64,17 +34,22 @@ class S3Output < Fluent::TimeSlicedOutput
   end
 
   def format(tag, time, record)
-    @formatter.call(tag, time, record)
+    time_str = @timef.format(time)
+    "#{time_str}\t#{tag}\t#{record.to_json}\n"
   end
 
   def write(chunk)
-    s3path = "#{@path}#{chunk.key}.gz"
+    i = 0
+    begin
+      s3path = "#{@path}#{chunk.key}_#{i}.gz"
+      i += 1
+    end while @bucket.objects[s3path].exists?
 
     tmp = Tempfile.new("s3-")
     w = Zlib::GzipWriter.new(tmp)
     begin
       chunk.write_to(w)
-      w.finish
+      w.close
       @bucket.objects[s3path].write(Pathname.new(tmp.path))
     ensure
       w.close rescue nil

metadata

@@ -1,105 +1,82 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
-  segments: 
-  - 0
-  - 2
-  - 0
-  version: 0.2.0
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-16 00:00:00 +09:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-11-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: fluentd
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70187303391020 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 55
-        segments: 
-        - 0
-        - 10
-        - 0
+      - !ruby/object:Gem::Version
         version: 0.10.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: aws-sdk
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70187303391020
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &70187303539040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 21
-        segments: 
-        - 1
-        - 1
-        - 3
+      - !ruby/object:Gem::Version
         version: 1.1.3
   type: :runtime
-  version_requirements: *id002
-description: 
+  prerelease: false
+  version_requirements: *70187303539040
+description: Amazon S3 output plugin for Fluent event collector
 email: frsyuki@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files: []
+files:
+- AUTHORS
 - ChangeLog
+- Gemfile
 - README.rdoc
-files: 
-- AUTHORS
 - Rakefile
 - VERSION
+- fluent-plugin-s3.gemspec
 - lib/fluent/plugin/out_s3.rb
 - test/out_s3.rb
-- ChangeLog
-- README.rdoc
-has_rdoc: true
-homepage: http://fluent.github.com/
+homepage: https://github.com/fluent/fluent-plugin-s3
 licenses: []
-
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: -2936927257322544690
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
+      hash: -2936927257322544690
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Amazon S3 output plugin for Fluent event collector
-test_files: 
+test_files:
 - test/out_s3.rb