fluent-plugin-dynamodb 0.1.5

data/ChangeLog

@@ -0,0 +1,8 @@
+Release 0.1.5 - 2012/06/10
+
+  * First release
+
+Release 0.1.0 - 2012/06/09
+
+  * First commit
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,50 @@
+# Amazon DynamoDB output plugin for Fluent event collector
+
+##Installation
+
+    $ fluent-gem install fluent-plugin-dynamodb
+
+##Configuration
+
+
+###DynamoDB
+
+First of all, you need to create a table in DynamoDB. It's easy to create via Management Console.
+
+Specify table name, hash attribute name and throughput as you like. fluent-plugin-dynamodb will load your table schema and write event-stream out to your table.
+
+*currently supports only table with a primary key which has a string hash-key. (hash and range key is not supported.)*
+
+### Fluentd
+
+    <match dynamodb.**>
+      type dynamodb
+      aws_key_id AWS_ACCESS_KEY
+      aws_sec_key AWS_SECRET_ACCESS_KEY
+      proxy_uri http://user:password@192.168.0.250:3128/
+      dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
+      dynamo_db_table access_log
+    </match>
+
+ * **aws\_key\_id (required)** - AWS access key id.
+ * **aws\_sec\_key (required)** - AWS secret key.
+ * **proxy_uri (optional)** - your proxy url.
+ * **dynamo\_db\_endpoint (required)** - end point of dynamodb. see  [Regions and Endpoints](http://docs.amazonwebservices.com/general/latest/gr/rande.html#ddb_region)
+ * **dynamo\_db\_table (required)** - table name of dynamodb.
+
+##TODO
+
+ * auto-create table
+ * tag_mapped
+ * Multiprocessing 
+
+##Copyright
+
+<table> 
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2012- Takashi Matsuno</td>
+  </tr>
+  <tr>
+    <td>License</td><td>Apache License, Version 2.0</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.5

data/fluent-plugin-dynamodb.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-dynamodb"
+  gem.description = "Amazon DynamoDB output plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/fluent/fluent-plugin-dynamodb"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Takashi Matsuno"]
+  gem.email       = "g0n5uk3@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "aws-sdk", "~> 1.5.2"
+  gem.add_dependency "uuidtools", "~> 2.1.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_dynamodb.rb

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+module Fluent
+
+
+class DynamoDBOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('dynamodb', self)
+
+  BATCHWRITE_ITEM_LIMIT = 25
+  BATCHWRITE_CONTENT_SIZE_LIMIT = 1024*1024
+
+  def initialize
+    super
+    require 'aws-sdk'
+    require 'msgpack'
+    require 'time'
+    require 'uuidtools'
+  end
+
+  config_param :aws_key_id, :string
+  config_param :aws_sec_key, :string
+  config_param :proxy_uri, :string, :default => nil
+  config_param :dynamo_db_table, :string
+  config_param :dynamo_db_endpoint, :string, :default => nil
+  config_param :time_format, :string, :default => nil
+
+  def configure(conf)
+    super
+
+    @timef = TimeFormatter.new(@time_format, @localtime)
+  end
+
+  def start
+    super
+    options = {
+      :access_key_id      => @aws_key_id,
+      :secret_access_key  => @aws_sec_key,
+      :dynamo_db_endpoint => @dynamo_db_endpoint,
+    }
+    options[:proxy_uri] = @proxy_uri if @proxy_uri
+
+    begin
+      restart_session(options)
+      valid_table(@dynamo_db_table)
+    rescue ConfigError => e
+      $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
+      exit!
+    rescue Exception => e
+      $log.fatal "UnknownError: '#{e}'"
+      exit!
+    end
+  end
+
+  def restart_session(options)
+    config = AWS.config(options)
+    @batch = AWS::DynamoDB::BatchWrite.new(config)
+    @dynamo_db = AWS::DynamoDB.new(options)
+  end
+
+  def valid_table(table_name)
+    table = @dynamo_db.tables[table_name]
+    table.load_schema
+    raise ConfigError, "Currently composite table is not supported." if table.has_range_key?
+    @hash_key_value = table.hash_key.name
+  end
+
+  def format(tag, time, record)
+    if !record.key?(@hash_key_value)
+      record[@hash_key_value] = UUIDTools::UUID.timestamp_create.to_s
+    end
+
+    [time, record].to_msgpack
+  end
+
+  def write(chunk)
+    records = collect_records(chunk)
+    item_num = item_size = 0
+    records.each {|record|
+      item_num += 1
+      item_size += record.to_json.length # FIXME: heuristic
+      if item_num >= BATCHWRITE_ITEM_LIMIT || item_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
+        @batch.process!
+        item_num = item_size = 0
+      end
+      @batch.put(@dynamo_db_table, [record])
+    }
+    @batch.process!
+  end
+
+  def collect_records(chunk)
+    records = []
+    chunk.msgpack_each { |time, record|
+      record['time'] = @timef.format(time)
+      records << record
+    }
+    records
+  end
+
+end
+
+
+end

data/test/out_dynamodb.rb

@@ -0,0 +1,59 @@
+require 'fluent/test'
+require 'fluent/plugin/out_dynamodb'
+
+class DynamoDBOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    aws_key_id test_key_id
+    aws_sec_key test_sec_key
+    dynamo_db_table test_table
+    dynamo_db_endpoint test.endpoint
+    utc
+    buffer_type memory
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::DynamoDBOutput) do
+      def write(chunk)
+        chunk.read
+      end
+    end.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'test_key_id', d.instance.aws_key_id
+    assert_equal 'test_sec_key', d.instance.aws_sec_key
+    assert_equal 'test_table', d.instance.dynamo_db_table
+    assert_equal 'test.endpoint', d.instance.dynamo_db_endpoint
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format([time, {'a' => 1}].to_msgpack)
+    d.expect_format([time, {'a' => 2}].to_msgpack)
+
+    d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    data = d.run
+
+    assert_equal [time, {'a' => 1}].to_msgpack + [time, {'a' => 2}].to_msgpack, data
+  end
+
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: fluent-plugin-dynamodb
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.5
+platform: ruby
+authors: 
+- Takashi Matsuno
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-06-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fluentd
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.10.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.5.2
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: uuidtools
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.1.0
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.2
+  type: :development
+  version_requirements: *id004
+description: Amazon DynamoDB output plugin for Fluent event collector
+email: g0n5uk3@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-dynamodb.gemspec
+- lib/fluent/plugin/out_dynamodb.rb
+- test/out_dynamodb.rb
+homepage: https://github.com/fluent/fluent-plugin-dynamodb
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Amazon DynamoDB output plugin for Fluent event collector
+test_files: 
+- test/out_dynamodb.rb