fluent-plugin-couch-sharded 0.6.0

data/AUTHORS

@@ -0,0 +1 @@
+Yudai Odagiri <ixixizko _at_ gmail.com>

data/README.rdoc

@@ -0,0 +1,58 @@
+= CouchDB output plugin for Fluentd
+
+fluent-plugin-couch provides output plugins for {Fluentd}[http://fluentd.org/].
+
+= Installation
+
+== Gems
+
+The gem is hosted at {Rubygems.org}[http://rubygems.org]. You can install the gem as follows:
+
+    $ fluent-gem install fluent-plugin-couch-sharded
+
+== Component
+
+=== CouchDBOutput
+
+Store fluent-event as CouchDB Document to CouchDB database.
+
+This is a forked version to add extra features.
+
+== Configuration
+
+=== CouchOutput
+
+    <match couch.**>
+      type couch
+      database fluent
+
+      # following attibutes are optional
+
+      host fluenter                #default:localhost
+      port 10000                   #default:5984
+      protocol https               #default:http
+
+      update_docs true             #default:false
+      doc_key_field doc_id         #default:nil
+      doc_key_jsonpath $.event.key #default:nil
+
+      refresh_view_index viewname  #default:nil
+
+      # for Basic Authentication (optional)
+      user odagiri
+      password yudai
+
+      # Other buffer configurations here
+    </match>
+
+== TODO
+=== More configuration
+=== Feature
+*
+
+== Tool
+
+== Copyright
+
+Copyright:: Copyright (c) 2011- Yudai Odagiri
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,33 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "fluent-plugin-couch-sharded"
+    gemspec.summary = "CouchDB output plugin for Fluentd event collector, forked to add 'sharding' features"
+    gemspec.author = "Yudai Odagiri,James Birmingham"
+    gemspec.email = "ixixizko@gmail.com"
+    gemspec.homepage = "http://github.com/ixixi/fluent-plugin-couch"
+    gemspec.has_rdoc = false
+    gemspec.require_paths = ["lib"]
+    gemspec.add_dependency "fluentd", "~> 0.10.0"
+    gemspec.add_dependency "couchrest", "~> 1.1.2"
+    gemspec.add_dependency "jsonpath", "~> 0.4.2"
+    gemspec.test_files = Dir["test/**/*.rb"]
+    gemspec.files = Dir["lib/**/*", "test/**/*.rb"] + %w[VERSION AUTHORS Rakefile]
+    gemspec.executables = []
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/*_test.rb']
+  t.ruby_opts = ['-rubygems'] if defined? Gem
+  t.ruby_opts << '-I.'
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.6.0

data/lib/fluent/plugin/out_couch.rb

@@ -0,0 +1,208 @@
+module Fluent
+	class CouchOutput < BufferedOutput
+		include SetTagKeyMixin
+		config_set_default :include_tag_key, false
+
+		include SetTimeKeyMixin
+		config_set_default :include_time_key, true
+
+		Fluent::Plugin.register_output('couch', self)
+
+		config_param :database, :string
+
+		config_param :host, :string, :default => 'localhost'
+		config_param :port, :string, :default => '5984'
+		config_param :protocol, :string, :default => 'http'
+
+		config_param :refresh_view_index, :string, :default => nil
+
+		config_param :user, :string, :default => nil
+		config_param :password, :string, :default => nil
+
+		config_param :update_docs, :bool, :default => false
+		config_param :doc_key_field, :string, :default => nil
+		config_param :doc_key_jsonpath, :string, :default => nil
+		config_param :db_key, :string, :default => nil
+		config_param :db_per, :string, :default => nil
+		config_param :w, :integer, :default => 1 #need to add support to couchrest for this
+		#
+		#
+		def initialize
+			super
+
+			require 'msgpack'
+			require 'jsonpath'
+			Encoding.default_internal = 'UTF-8'
+			require 'couchrest'
+			Encoding.default_internal = 'ASCII-8BIT'
+		end
+		#
+		#
+		def configure(conf)
+			super
+		end
+		#
+		#
+		def start
+			super
+
+			if @db_key or @db_per
+				raise "update_docs is not compatible with db_key or db_per yet!" if @update_docs
+				raise "refresh_view_index is not compatible with db_key or db_per yet!" if @refresh_view_index
+			end
+
+			@account = "#{@user}:#{@password}@" if @user && @password
+
+			@db = {}
+
+			#
+			# todo : doesn't work
+			#
+			if @db_per and @db_key
+				@write = self.method :write_by_key_per
+			elsif @db_per
+				raise "todo: do not currenty support db_per without db_key"
+			elsif @db_key
+				@write = self.method :write_by_key
+			else
+				@write = self.method :write_default
+			end
+
+			if not @db_key or @db_per
+				@db[:default] = CouchRest.database!("#{@protocol}://#{@account}#{@host}:#{@port}/#{@database}")
+			end
+
+			@views = []
+
+			if @refresh_view_index
+				begin
+					@db[:default].get("_design/#{@refresh_view_index}")['views'].each do |view_name, func|
+						@views.push([@refresh_view_index, view_name])
+					end
+				rescue
+					$log.error 'design document not found!'
+				end
+			end
+		end
+		#
+		#
+		def shutdown
+			super
+		end
+		#
+		#
+		def format(tag, time, record)
+			record.to_msgpack
+		end
+		#
+		# todo : this sucks
+		#
+		def write(chunk)
+			if @db_per and @db_key
+				write_by_key_per chunk
+			elsif @db_per
+				raise "todo: do not currenty support db_per without db_key"
+			elsif @db_key
+				write_by_key chunk
+			else
+				write_default chunk
+			end
+		end
+		def write_default(chunk)
+			records = []
+
+			chunk.msgpack_each do |record|
+
+				id = record[@doc_key_field]
+				id = JsonPath.new(@doc_key_jsonpath).first(record) if id.nil? && !@doc_key_jsonpath.nil?
+				record['_id'] = id unless id.nil?
+				records << record
+			end
+
+			if @update_docs
+				update_docs(records)
+			else
+				@db[:default].bulk_save(records)
+			end
+
+			update_view_index
+		end
+		#
+		#
+		def write_by_key_per(chunk)
+			records = {}
+
+			chunk.msgpack_each do |record|
+				id = record[@doc_key_field]
+				id = JsonPath.new(@doc_key_jsonpath).first(record) if id.nil? && !@doc_key_jsonpath.nil?
+				record['_id'] = id unless id.nil?
+
+				date = record['dt']
+
+				key = "#{record[@db_key].downcase}_#{date[0]}#{date[1].to_s.rjust(2, '0')}#{date[2].to_s.rjust(2, '0')}"
+
+				records[key] = [] unless records[key]
+				records[key] << record
+			end
+
+			records.each_pair do |key, data|
+				if not @db[key]
+					@db[key] = CouchRest.database!("#{@protocol}://#{@account}#{@host}:#{@port}/#{@database+"_"+key}")
+				end
+
+				@db[key].bulk_save(data)
+			end
+
+		end
+		#
+		#
+		def write_by_key(chunk)
+			records ={}
+
+			chunk.msgpack_each do |record|
+				id = record[@doc_key_field]
+				id = JsonPath.new(@doc_key_jsonpath).first(record) if id.nil? && !@doc_key_jsonpath.nil?
+				record['_id'] = id unless id.nil?
+
+				key =  record[@db_key]
+				records[key] = [] unless records[key]
+				records[key] << record
+			end
+
+			records.each_pair do |key, data|
+				if not @db[key]
+					@db[key] = CouchRest.database!("#{@protocol}://#{@account}#{@host}:#{@port}/#{@database+"_"+key}")
+				end
+
+				@db[key].bulk_save(data)
+			end
+
+		end
+		#
+		#
+		def update_docs(records)
+
+			if records.length > 0
+				records.each do |record|
+					doc = nil
+
+					begin
+						doc = @db[:default].get(record['_id'])
+					rescue
+					end
+
+					record['_rev']=doc['_rev'] unless doc.nil?
+					$log.debug record
+					@db[:default].save_doc(record)
+				end
+			end
+		end
+		#
+		#
+		def update_view_index()
+			@views.each do |design, view|
+				@db[:default].view("#{design}/#{view}", {'limit' => '0'})
+			end
+		end
+	end
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-couch-sharded
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+  prerelease: 
+platform: ruby
+authors:
+- Yudai Odagiri,James Birmingham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-couch-sharded
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: couchrest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+description: 
+email: ixixizko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- AUTHORS
+- Rakefile
+- VERSION
+- lib/fluent/plugin/out_couch.rb
+- README.rdoc
+homepage: http://github.com/ixixi/fluent-plugin-couch
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4573471804670427138
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: CouchDB output plugin for Fluentd event collector, forked to add 'sharding'
+  features
+test_files: []