fluent-plugin-cassandra 0.0.1 → 0.0.2

data/README.rdoc

@@ -1,13 +1,14 @@
 = fluent-plugin-cassandra
 
-= MongoDB output plugin for Fluent
+= Cassandra output plugin for Fluent
 
 == Configuratin
 
-    <match **>                              # cassandra's key is tags + event.time.
+    <match cassandra.**>                              # cassandra's key is tags + event.time.
       type cassandra                        # 
-      keyspace Keyspace1                    # keyspace
-      columnfamily Standard1                # Columnfamily
+      localtime                             # set localtime for key
+      keyspace FluentdLoggers               # keyspace
+      columnfamily EventLogger              # Columnfamily
 
       # following attibutes are optional
       host localhost                        # cassandra's hostname. default localhost
@@ -20,9 +21,36 @@
 === More configuration
 
 - Create Keyspace
+
+create keyspace FluentdLoggers
+    with strategy_options=[{replication_factor:1}]
+    and placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy';
+
+
 - Create Columnfamily
+#example CF
+
+use FluentdLoggers;
+
+create column family EventLogger
+    with comparator = UTF8Type
+    and column_metadata = [{
+        column_name : tag,
+        validation_class: UTF8Type,
+        index_name : tag_idx,
+        index_type: KEYS},{
+        column_name : time,
+        validation_class : BytesType,
+        index_name : time_idx,
+        index_type: KEYS}
+    ];
+
 - etc
 
+put plugin
+cp -ip ${GEM PATHS}/gem/fluent-plugin-cassandra-0.0.2/lib/fluent/plugin/out_cassandra.rb /etc/fluent/plugin/.
+
+
 === Infer collection name
 
 Fluent tag and event.time is key in Cassandra.

data/Rakefile

@@ -21,26 +21,28 @@ Jeweler::Tasks.new do |gem|
   gem.description = "Cassandra output plugin for Fluent event collector"
   gem.email = "tomitakazutaka@gmail.com"
   gem.authors = ["Kazutaka Tomita"]
-  gem.add_dependency "fluent", "~> 0.9.14"
-  gem.add_dependency "cassandra", "~> 0.12.1"
+  gem.version = '0.0.2'
+  gem.add_dependency("fluentd", "~> 0.10.0")
+  gem.add_dependency("cassandra", "~> 0.12.1")
+  gem.files = FileList['lib/fluent/plugin/out_cassandra.rb',  '[A-Z]*'].to_a
+  gem.bindir = '/etc/fluent/plugin'
 
-  # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+#  test.libs << 'lib' << 'test'
+#  test.pattern = 'test/**/test_*.rb'
+#  test.verbose = true
 end
 
 require 'rcov/rcovtask'
 Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
+#  test.libs << 'test'
+#  test.pattern = 'test/**/test_*.rb'
+#  test.verbose = true
+#  test.rcov_opts << '--exclude "gems/*"'
 end
 
 task :default => :test
@@ -52,5 +54,4 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "fluent-plugin-cassandra #{version}"
   rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/fluent/plugin/*.rb')
 end

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/fluent/plugin/out_cassandra.rb

@@ -3,6 +3,12 @@ module Fluent
 
 class CassandraOutput < BufferedOutput
   Fluent::Plugin.register_output('cassandra', self)
+  include SetTimeKeyMixin
+  include SetTagKeyMixin
+  config_set_default :include_time_key, true
+  config_set_default :include_tag_key, true
+  config_set_default :time_format, "%Y%m%d%H%M%S"
+
 
   def initialize
     super
@@ -13,8 +19,8 @@ class CassandraOutput < BufferedOutput
   def configure(conf)
     super
 
-    raise ConfigError, "'Keyspace' parameter is required on file output"   unless @keyspace = conf['keyspace']
-    raise ConfigError, "'ColumnFamily' parameter is required on file output"   unless @columnfamily = conf['columnfamily']
+    raise ConfigError, "'Keyspace' parameter is required on cassandra output"   unless @keyspace = conf['keyspace']
+    raise ConfigError, "'ColumnFamily' parameter is required on cassandra output"   unless @columnfamily = conf['columnfamily']
 
     @host = conf.has_key?('host') ? conf['host'] : 'localhost'
     @port = conf.has_key?('port') ? conf['port'] : 9160
@@ -29,20 +35,17 @@ class CassandraOutput < BufferedOutput
     super
   end
 
-  def format(tag, event)
-    [tag,event.time,event.record].to_msgpack
+  def format(tag, time,record)
+    record.to_msgpack
   end
 
   def write(chunk)
-    chunk.open { |io|
-      begin
-        MessagePack::Unpacker.new(io).each { |record|
-            c_key = record[0] + "_" + record[1].to_s
-            @connection.insert(@columnfamily,c_key,record[2])
-        }
-      rescue EOFError
-        # EOFError always occured when reached end of chunk.
-      end
+    chunk.msgpack_each  { |record|
+            @connection.insert(
+                    @columnfamily,
+                    record["tag"] + "_" + record["time"].to_s,
+                    record
+                    )
     }
   end
 end

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cassandra
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
 - Kazutaka Tomita
 autorequire: 
-bindir: bin
+bindir: /etc/fluent/plugin
 cert_chain: []
-date: 2011-10-03 00:00:00.000000000Z
+date: 2011-10-22 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &16887040 !ruby/object:Gem::Requirement
+  requirement: &24711020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *16887040
+  version_requirements: *24711020
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &16886520 !ruby/object:Gem::Requirement
+  requirement: &24709860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *16886520
+  version_requirements: *24709860
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &16886040 !ruby/object:Gem::Requirement
+  requirement: &24676540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *16886040
+  version_requirements: *24676540
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &16885560 !ruby/object:Gem::Requirement
+  requirement: &24674980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,21 +54,21 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *16885560
+  version_requirements: *24674980
 - !ruby/object:Gem::Dependency
-  name: fluent
-  requirement: &16885080 !ruby/object:Gem::Requirement
+  name: fluentd
+  requirement: &24673780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.14
+        version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *16885080
+  version_requirements: *24673780
 - !ruby/object:Gem::Dependency
   name: cassandra
-  requirement: &16884600 !ruby/object:Gem::Requirement
+  requirement: &24672460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.12.1
   type: :runtime
   prerelease: false
-  version_requirements: *16884600
+  version_requirements: *24672460
 description: Cassandra output plugin for Fluent event collector
 email: tomitakazutaka@gmail.com
 executables: []
@@ -85,14 +85,12 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
 files:
-- .document
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.rdoc
 - Rakefile
 - VERSION
-- fluent-plugin-cassandra.gemspec
 - lib/fluent/plugin/out_cassandra.rb
 homepage: http://github.com/tomitakazutaka/fluent-plugin-cassandra
 licenses:
@@ -109,7 +107,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3147957507486541088
+      hash: 1258677813083483526
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/.document

@@ -1,5 +0,0 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt

data/fluent-plugin-cassandra.gemspec

@@ -1,63 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = "fluent-plugin-cassandra"
-  s.version = "0.0.1"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Kazutaka Tomita"]
-  s.date = "2011-10-03"
-  s.description = "Cassandra output plugin for Fluent event collector"
-  s.email = "tomitakazutaka@gmail.com"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-    "Gemfile",
-    "Gemfile.lock",
-    "LICENSE.txt",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "fluent-plugin-cassandra.gemspec",
-    "lib/fluent/plugin/out_cassandra.rb"
-  ]
-  s.homepage = "http://github.com/tomitakazutaka/fluent-plugin-cassandra"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "Cassandra output plugin for Fluent event collector"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_runtime_dependency(%q<fluent>, ["~> 0.9.14"])
-      s.add_runtime_dependency(%q<cassandra>, ["~> 0.12.1"])
-    else
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<fluent>, ["~> 0.9.14"])
-      s.add_dependency(%q<cassandra>, ["~> 0.12.1"])
-    end
-  else
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<fluent>, ["~> 0.9.14"])
-    s.add_dependency(%q<cassandra>, ["~> 0.12.1"])
-  end
-end
-