fluent-plugin-aerospike-cluster 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f7d9aa2ba159d94825b57d80a1b54f2be9ec854c
+  data.tar.gz: b11901411ce2c033061cc9163aa4f94efdaa47a3
+SHA512:
+  metadata.gz: b08a73fe71b49c5595e3c89c54fe94e9bd47bb1c01979c5082a55811f7c84ec19c69b1ab153e9839f80e15459a1b863a584c8709e5adc36200a4904aa01f5836
+  data.tar.gz: d4c59b40489e76f9e84f1400740a15de7ae61987d0bdb7d39df846d128d78be3a23b6191a6efbf3d6c265149a3c1d4ecd58eef57cc944ab63c5d34756d590d32

data/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.0.0
+before_install: gem install bundler -v 1.12.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-aerospike-cluster.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (C) 2016 fluent-plugin-aerospike-cluster authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,30 @@
+# fluent-plugin-aerospike-cluster
+
+fluent output plugin for aerospike.
+
+
+## Parameters
+
+param                | type     | value                                                              | default
+---------------------|----------|--------------------------------------------------------------------|---------------
+hosts                | string   | Aerospike host:port list. (comma separated)                        | 127.0.0.1:3000
+username             | string   | Aerospike username. (not implemented yet)                          |
+password             | string   | Aerospike password. (not implemented yet)                          |
+timeout              | float    | Connection timeout second.                                         | 1.0
+connection_queue_size| integer  | Size of the connection queue cache.                                | 64
+tend_interval        | float    | Tend interval in milliseconds; determines the                      interval at which the client checks for cluster state changes. Minimum interval is 10ms.  | 1000
+namespace            | string   | Aerospike namespace.                                               |
+set                  | string   | Aerospike set.                                                     |
+keys                 | string   | Keys for key of record.                                            |
+record_keys          | string   | Keys for Record values. (comma separated)                          |
+send_key             | bool     | Send user defined key in addition to hash digest on a record put.  | true
+ttl                  | integer  | Record ttl seconds. (-1: never expire, 0: default namespace's ttl) | 0
+record_exists_action | string   | Qualify how to handle writes where the record already exists.  update update_only replace replace_only  |
+
+## Another Parameters
+
+- include_tag_key
+- tag_key
+- include_time_key
+- time_key
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/fluent-plugin-aerospike-cluster.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-aerospike-cluster"
+  spec.version       = "0.0.1"
+  spec.authors       = ["kadzuya"]
+  spec.email         = ["kadzuya@gmail.com"]
+
+  spec.summary       = %q{fluent plugin to insert Aerospike.}
+  spec.description   = %q{fluent plugin to insert Aerospike.}
+  spec.homepage      = "http://github.com/kadzuya/fluent-plugin-aerospike-cluster"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "aerospike", '>= 1.0.0'
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/fluent/plugin/out_aerospike_cluster.rb

@@ -0,0 +1,115 @@
+module Fluent
+  class AerospikeClusterOutput < BufferedOutput
+    Fluent::Plugin.register_output('aerospike_cluster', self)
+    include Fluent::SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include Fluent::SetTimeKeyMixin
+    config_set_default :include_time_key, true
+
+    config_param :hosts, :string, :default => '127.0.0.1:3000',
+                 :desc => "Aerospike host:port list. (comma separated)"
+    config_param :username, :string, :default => nil,
+                 :desc => "Aerospike username. (not implemented yet)"
+    config_param :password, :string, :default => nil,
+                 :desc => "Aerospike password. (not implemented yet)"
+    config_param :timeout, :float, :default => 1.0,
+                 :desc => "Connection timeout second."
+    config_param :connection_queue_size, :integer, :default => 64,
+                 :desc => "Size of the connection queue cache."
+    config_param :tend_interval, :float, :default => 1000,
+                 :desc => "Tend interval in milliseconds; determines the interval at which the client checks for cluster state changes. Minimum interval is 10ms."
+    config_param :namespace, :string,
+                 :desc => "Aerospike namespace."
+    config_param :set, :string,
+                 :desc => "Aerospike set."
+    config_param :keys, :string,
+                 :desc => "Keys for key of record."
+    config_param :record_keys, :string,
+                 :desc => "Keys for record values. (comma separated)"
+    config_param :send_key, :bool, :default => true,
+                 :desc => "Send user defined key in addition to hash digest on a record put."
+    config_param :ttl, :integer, :default => 0,
+                 :desc => "Record ttl seconds. (-1: never expire, 0: default namespace's ttl)"
+    config_param :record_exists_action, :string, :default => nil,
+                 :desc => <<DESC
+Qualify how to handle writes where the record already exists.
+update update_only replace replace_only
+DESC
+    attr_accessor :client_policy, :write_policy
+
+    def configure(conf)
+      super
+      @hosts = @hosts.split(/\s*,\s*/).map {|host|
+        host = host.split(":")
+        Aerospike::Host.new(host[0], host[1].nil? ? 3000 : host[1].to_i)
+      }
+      @client_policy = Aerospike::ClientPolicy.new({
+        :user                  => @username,
+        :password              => @password,
+        :timeout               => @timeout,
+        :connection_queue_size => @connection_queue_size,
+        :tend_interval         => @tend_interval
+      })
+      @write_policy = Aerospike::WritePolicy.new({
+        :record_exists_action  => @record_exists_action.nil? ? nil : Aerospike::RecordExistsAction.const_get(@record_exists_action.upcase.to_sym),
+        :generation_policy     => nil,
+        :generation            => nil,
+        :ttl                   => @ttl,
+        :send_key              => @send_key,
+        :commit_level          => nil
+      })
+      @keys = @keys.split(/\s*,\s*/)
+      @record_keys = @record_keys.split(/\s*,\s*/)
+    end
+
+    def initialize
+      super
+      require 'aerospike'
+    end
+
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def client
+      @handler ||= Aerospike::Client.new_many(@hosts, @client_policy)
+    end
+
+    def start
+      super
+      client
+    end
+
+    def shutdown
+      @handler.close
+      super
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      log.debug "<<<<<<<<<<<<<<{Aerospike write chunk}<<<<<<<<<<<<<"
+      chunk.msgpack_each do |tag, time, record|
+        # Key
+        if @keys.length == 1
+          key = Aerospike::Key.new(@namespace, @set, record[@keys[0]])
+        else
+          key = Aerospike::Key.new(@namespace, @set, @keys.map {|col| record[col]})
+        end
+        # Value
+        if @record_keys.length > 0
+          record = record.select {|key, _| @record_keys.include? key }
+        end
+
+        log.debug "key:   #{key.to_s}"
+        log.debug "value: #{record.to_s}"
+        @handler.put(key, record, @write_policy)
+      end
+      log.debug ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    end
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aerospike-cluster
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kadzuya
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aerospike
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin to insert Aerospike.
+email:
+- kadzuya@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-aerospike-cluster.gemspec
+- lib/fluent/plugin/out_aerospike_cluster.rb
+homepage: http://github.com/kadzuya/fluent-plugin-aerospike-cluster
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: fluent plugin to insert Aerospike.
+test_files: []