fluent-plugin-aerospike 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 070dedcc9f21487d7958469da973cbd1371511cd
+  data.tar.gz: aa1e94e01a83b7d1efc49897cb55548c4e1cc2c7
+SHA512:
+  metadata.gz: 8e9cb50464d0ed21d597751c6bada52653f5fc6c9cdf83ef02f41a0b8e09b94bf7fb6ae443bbabe5066e2ff1bd26851dab41d570a0df8eb9e1ec2c88b5559d75
+  data.tar.gz: 086e9860fdb88b1700065db5748fecf419ed08064877d6faeb1e926b70c9fdbda48d58b7d2bc4afe9b4bf52aff2d2b6605e2fa5220b90b30542f47f6c848cf65

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-aerospike.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2014- Hatayama Hideharu
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,60 @@
+# Fluent::Plugin::Aerospike
+
+Fluent BufferedOutput plugin for Aerospike
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-aerospike'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-aerospike
+
+## Configuration
+config             | default         | info
+-------------------|-----------------|-----
+address            | 127.0.0.1:3000  | address of Aerospike
+namespace          | test            | namespace name of Aerospike
+set                | fluent ${tag}   | set name of Aerospike
+ttl                | 0               | time-to-live: If ttl = 0, the default ttl specified on the server side will be applied
+time_key           |                 | this time_key field or time of record is used
+time_format        |                 | (with time_key) the value of time_key will be parsed with this format
+
+## Samples
+### sample: access log
+config
+
+    <match ***>
+      type aerospike
+      time_key request_time
+      time_format %d/%b/%Y:%H:%M:%S %Z
+    </match>
+
+log
+
+    {"request_time":"24/Dec/2014:23:59:59 +0700","code":"100","response_time":"1000","site":"git"}
+
+Aerospike
+
+    aql> select * from test
+    +---------------+------------------------------+-------+---------------+
+    | site          | request_time                 | code  | response_time |
+    +---------------+------------------------------+-------+---------------+
+    | "git"         | "24/Dec/2014:23:59:59 +0700" | "100" | "1000"        |
+    +---------------+------------------------------+-------+---------------+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-aerospike/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/fluent-plugin-aerospike.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-aerospike"
+  spec.version       = "0.0.0"
+  spec.authors       = ["Hatayama Hideharu"]
+  spec.email         = ["h.hiddy@gmail.com"]
+  spec.description   = %q{Fluent BufferedOutput plugin for Aerospike}
+  spec.summary       = spec.description
+  spec.homepage      = "https://bitbucket.org/hidepiy/fluent-plugin-aerospike"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "fluentd", "~> 0"
+  spec.add_dependency "aerospike", "0.1.3"
+  spec.add_dependency "uuidtools", "~> 2.1", ">= 2.1.5"
+end
+

data/lib/fluent/plugin/out_aerospike.rb

@@ -0,0 +1,88 @@
+module Fluent
+  class AerospikeOutput < BufferedOutput
+    # First, register the plugin. NAME is the name of this plugin
+    # and identifies the plugin in the configuration file.
+    Plugin.register_output('aerospike', self)
+
+    # config_param defines a parameter. You can refer a parameter via @path instance variable
+    # Without :default, a parameter is required.
+    
+    # Aerospike client settings
+    config_param :address,    :string, :default => '127.0.0.1:3000'
+    config_param :namespace, :string, :default => 'test'
+    config_param :set, :string, :default => nil 
+    config_param :time_key, :string, :default => nil
+    config_param :time_format, :string, :default => nil
+    config_param :ttl, :integer, :default => 0
+
+    attr_reader :write_policy
+
+    def initialize
+      super
+      require 'aerospike'
+      require 'msgpack'
+      require 'uuidtools'
+    end
+
+    # This method is called before starting.
+    # 'conf' is a Hash that includes configuration parameters.
+    # If the configuration is invalid, raise Fluent::ConfigError.
+    def configure(conf)
+      super
+      # You can also refer raw parameter via conf[name].
+      write_policy = Aerospike::WritePolicy.new(
+        Aerospike::RecordExistsAction::CREATE_ONLY,
+        nil,
+        nil,
+        @ttl,
+        nil
+      )
+    end
+
+    # This method is called when starting.
+    # Open sockets or files here.
+    def start
+      super
+      @client = get_client(@address)
+    end
+
+    # This method is called when shutting down.
+    # Shutdown the thread and close sockets or files here.
+    def shutdown
+      @client.close
+      super
+    end
+
+    # This method is called when an event reaches to Fluentd.
+    # Convert the event to a raw string.
+    # [tag, time, record].to_json + "\n"
+    # Alternatively, use msgpack to serialize the object.
+    # [tag, time, record].to_msgpack
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # This method is called every flush interval. Write the buffer chunk
+    # to files or databases here.
+    # 'chunk' is a buffer chunk that includes multiple formatted
+    # events. You can use 'data = chunk.read' to get all events and
+    # 'chunk.open {|io| ... }' to get IO objects.
+    #
+    # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
+    # Optionally, you can use chunk.msgpack_each to deserialize objects.
+    def write(chunk)
+      chunk.msgpack_each {|(tag,time,record)|
+#         key_s = "#{time_key.nil? time : record[time_key]}-#{UUIDTools::UUID.random_create}"
+        key_s = "#{record[@time_key] || time}-#{UUIDTools::UUID.random_create}"
+        set_s = @set || tag
+        key = Aerospike::Key.new(@namespace, set_s, key_s)
+        @client.put(key, record, write_policy)
+      }
+    end
+
+    def get_client(address)
+      host_port = address.split(':', 2)
+      return Aerospike::Client.new(host_port[0], host_port[1])
+    end
+  end
+end

data/test/fluent/plugin/out_aerospike_test.rb

@@ -0,0 +1,70 @@
+require 'helper'
+
+class DataCounterOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONFIG = %[
+  ]
+
+  DEFAULT_TAG = 'default_tag'
+
+  def create_driver(conf = DEFAULT_CONFIG, tag = DEFAULT_TAG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::AerospikeOutput, tag).configure(conf)
+  end
+
+  def test_configure_default
+    d = create_driver
+    assert_equal '127.0.0.1:3000', d.instance.address
+    assert_equal 'test', d.instance.namespace
+    assert_equal nil, d.instance.set
+    assert_equal nil, d.instance.time_key
+    assert_equal nil, d.instance.time_format
+    assert_equal 0, d.instance.ttl
+  end
+
+  def test_configure
+    d = create_driver(DEFAULT_CONFIG + %[
+      address aerosmith:1234
+      namespace rock
+      set boston
+      ttl 1
+    ])
+    assert_equal 'aerosmith:1234', d.instance.address
+    assert_equal 'rock', d.instance.namespace
+    assert_equal 'boston', d.instance.set
+    assert_equal 1, d.instance.ttl
+  end
+
+  def test_format
+    d = create_driver
+    time = Time.parse("2112-09-03 01:23:45 UTC")
+    d.emit({"gerogero" => "Let's get Rocking!", "site" => "yapoo"}, time)
+    d.emit({"gerogero" => "Let's get Rocking!", "site" => "geegero"}, time)
+    d.expect_format "\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA5yapoo\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA7geegero"
+    # d.run
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    # d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+    # ### FileOutput#write returns path
+    # path = d.run
+    # expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
+    # assert_equal expect_path, path
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_aerospike'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aerospike
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Hatayama Hideharu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aerospike
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
+description: Fluent BufferedOutput plugin for Aerospike
+email:
+- h.hiddy@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-aerospike.gemspec
+- lib/fluent/plugin/out_aerospike.rb
+- test/fluent/plugin/out_aerospike_test.rb
+- test/helper.rb
+homepage: https://bitbucket.org/hidepiy/fluent-plugin-aerospike
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Fluent BufferedOutput plugin for Aerospike
+test_files:
+- test/fluent/plugin/out_aerospike_test.rb
+- test/helper.rb