fluent-plugin-dynamodb-drc 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 569ec51668adffaf4ba084880612169126332238
+  data.tar.gz: 57d212b6852bb4761304745b274a9d8dd74878a4
+SHA512:
+  metadata.gz: 814a7c301ef4da1deeb0de0fcc57dba047a40b7f2d85ed7397e4d75fddc0b1752de606d3d64d192798879423febf040a21a11231bfef5099eccf125e3781510e
+  data.tar.gz: b936f092a2b12191e391ca5d81fb44b8a94c605063e3ece357f48a807995e324a7c5c770ee1fdad9bbadd676c1bac937cdba40550c5164716f707ee2944f9784

data/AUTHORS

@@ -0,0 +1,3 @@
+Takashi Matsuno
+Sadayuki Furuhashi
+Carlos Donderis

data/ChangeLog

@@ -0,0 +1,20 @@
+Release 0.1.8 - 2012/07/10
+
+  * Fix gem.homepage url 
+
+Release 0.1.7 - 2012/06/17
+
+  * Inherits DetachMultiProcessMixin
+
+Release 0.1.6 - 2012/06/12
+
+ * Optimized write(chunk) method not to collect all records in memory
+
+Release 0.1.5 - 2012/06/10
+
+  * First release
+
+Release 0.1.0 - 2012/06/09
+
+  * First commit
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,6 @@
+# Fixed version of Amazon DynamoDB output plugin for [Fluentd]
+
+##Installation
+
+    $ fluent-gem install fluent-plugin-dynamodb-drc
+

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/fluent-plugin-dynamodb-drc.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-dynamodb-drc"
+  gem.description = "Amazon DynamoDB output plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/kumapon/fluent-plugin-dynamodb-drc"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Takashi Matsuno", "Sadayuki Furuhashi", "CaDs"]
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "aws-sdk", ">= 1.5.2"
+  gem.add_dependency "uuidtools", "~> 2.1.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_dynamodb.rb

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+module Fluent
+
+
+class DynamoDBOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('dynamodb_drc', self)
+
+  include DetachMultiProcessMixin
+
+  BATCHWRITE_ITEM_LIMIT = 25
+  BATCHWRITE_CONTENT_SIZE_LIMIT = 1024*1024
+
+  def initialize
+    super
+    require 'aws-sdk'
+    require 'msgpack'
+    require 'time'
+    require 'uuidtools'
+  end
+
+  config_param :aws_key_id, :string, :default => nil
+  config_param :aws_sec_key, :string, :default => nil
+  config_param :proxy_uri, :string, :default => nil
+  config_param :dynamo_db_table, :string
+  config_param :dynamo_db_endpoint, :string, :default => nil
+  config_param :time_format, :string, :default => nil
+  config_param :detach_process, :integer, :default => 2
+
+  def configure(conf)
+    super
+
+    @timef = TimeFormatter.new(@time_format, @localtime)
+  end
+
+  def start
+    options = {}
+    if @aws_key_id && @aws_sec_key
+      options[:access_key_id] = @aws_key_id
+      options[:secret_access_key] = @aws_sec_key
+    end
+    options[:dynamo_db_endpoint] = @dynamo_db_endpoint
+    options[:proxy_uri] = @proxy_uri if @proxy_uri
+
+    detach_multi_process do
+      super
+
+      begin
+        restart_session(options)
+        valid_table(@dynamo_db_table)
+      rescue ConfigError => e
+        $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
+        exit!
+      rescue Exception => e
+        $log.fatal "UnknownError: '#{e}'"
+        exit!
+      end
+    end
+  end
+
+  def restart_session(options)
+    config = AWS.config(options)
+    @batch = AWS::DynamoDB::BatchWrite.new(config)
+    @dynamo_db = AWS::DynamoDB.new(options)
+  end
+
+  def valid_table(table_name)
+    table = @dynamo_db.tables[table_name]
+    table.load_schema
+    @hash_key = table.hash_key
+    @range_key = table.range_key unless table.simple_key?
+  end
+
+  def match_type!(key, record)
+    if key.type == :number
+      potential_value = record[key.name].to_i
+      if potential_value == 0
+        $log.fatal "Failed attempt to cast hash_key to Integer."
+      end
+      record[key.name] = potential_value
+    end
+  end
+
+  def format(tag, time, record)
+    if !record.key?(@hash_key.name)
+      record[@hash_key.name] = UUIDTools::UUID.timestamp_create.to_s
+    end
+    match_type!(@hash_key, record)
+
+    formatted_time = @timef.format(time)
+    if @range_key
+      if !record.key?(@range_key.name)
+        record[@range_key.name] = formatted_time
+      end
+      match_type!(@range_key, record)
+    end
+    record['time'] = formatted_time
+
+    record.to_msgpack
+  end
+
+  def write(chunk)
+    batch_size = 0
+    batch_records = []
+    chunk.msgpack_each {|record|
+      fixed_timestamp = "#{Time.zone.parse(record["time"]).to_i}#{rand(1000000000..9999999999)}"
+      record["timestamp"] = fixed_timestamp
+      batch_records << record
+      batch_size += record.to_json.length # FIXME: heuristic
+      if batch_records.size >= BATCHWRITE_ITEM_LIMIT || batch_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
+        batch_put_records(batch_records)
+        batch_records.clear
+        batch_size = 0
+      end
+    }
+    unless batch_records.empty?
+      batch_put_records(batch_records)
+    end
+  end
+
+  def batch_put_records(records)
+    @batch.put(@dynamo_db_table, records)
+    @batch.process!
+  end
+
+end
+
+
+end
+

data/test/out_dynamodb.rb

@@ -0,0 +1,59 @@
+require 'fluent/test'
+require 'fluent/plugin/out_dynamodb'
+
+class DynamoDBOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    aws_key_id test_key_id
+    aws_sec_key test_sec_key
+    dynamo_db_table test_table
+    dynamo_db_endpoint test.endpoint
+    utc
+    buffer_type memory
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::DynamoDBOutput) do
+      def write(chunk)
+        chunk.read
+      end
+    end.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'test_key_id', d.instance.aws_key_id
+    assert_equal 'test_sec_key', d.instance.aws_sec_key
+    assert_equal 'test_table', d.instance.dynamo_db_table
+    assert_equal 'test.endpoint', d.instance.dynamo_db_endpoint
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format([time, {'a' => 1}].to_msgpack)
+    d.expect_format([time, {'a' => 2}].to_msgpack)
+
+    d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    data = d.run
+
+    assert_equal [time, {'a' => 1}].to_msgpack + [time, {'a' => 2}].to_msgpack, data
+  end
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-dynamodb-drc
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Takashi Matsuno
+- Sadayuki Furuhashi
+- CaDs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Amazon DynamoDB output plugin for Fluent event collector
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- AUTHORS
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-dynamodb-drc.gemspec
+- lib/fluent/plugin/out_dynamodb.rb
+- test/out_dynamodb.rb
+homepage: https://github.com/kumapon/fluent-plugin-dynamodb-drc
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: Amazon DynamoDB output plugin for Fluent event collector
+test_files:
+- test/out_dynamodb.rb