fluent-plugin-hoop 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,31 @@
+# rcov generated
+coverage
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# For MacOS
+.DS_Store
+
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+
+# not to lock gems version, and for bundler
+Gemfile.lock
+vendor
+vendor/fluentd

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/fluentd"]
+	path = vendor/fluentd
+	url = git://github.com/fluent/fluentd.git

data/AUTHORS

@@ -0,0 +1 @@
+TAGOMORI Satoshi <tagomoris _at_ gmail.com>

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end
+
+gem "rdoc"
+

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2011 TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.rdoc

@@ -0,0 +1,79 @@
+= Hoop plugin for Fluentd
+
+== Component
+
+=== HoopOutput
+
+Store fluent-event as plain text to HDFS, over Hoop (HDFS http-fs).
+
+Hoop is originally written in Cloudera, and merged on Apache Hadoop 0.23 tree. See:
+
+[Apache Hadoop dev doc] https://github.com/apache/hadoop-common/blob/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
+[Cloudera Hoop doc (obsolete)] http://cloudera.github.com/hoop/docs/latest/index.html
+
+HoopOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
+
+- format whole data as serialized JSON, single attribute or separated multi attributes
+- include time as line header, or not
+- include tag as line header, or not
+- change field separator (default: TAB)
+
+== Configuration
+
+=== HoopOutput
+
+Minimal configuration (output: TAB separated time,tag,json-serialized-data and terminated with newline):
+
+    <match hoop.**>
+      type hoop
+      hoop_server hoop-server.local:14000
+    
+      # %Y %m %d %H %M %S are available as conversion specifications in path on hdfs
+      # If '%Y%m%d' specified, logs are sliced into per-day files automatically.
+      path /hoop/log-%Y%m/log-%Y%m%d.log
+    
+      # 'username' is used pseudo authentication, see http://cloudera.github.com/hoop/docs/latest/HttpRestApi.html
+      username hoopuser
+    </match>
+
+You will get output like below in hdfs file such as '/hoop/log-201112/log-20111231.log'
+
+    2011-12-31T13:14:15Z [TAB] hoop.foo.bar [TAB] {"field1":12345,"field2":"one two three four five","field3":"OK"} [terminated by newline]
+    2011-12-31T21:22:23Z [TAB] hoop.foo.val [TAB] {"field1":23456,"field2":"two three four five six","field3":"BAD"} [terminated by newline]
+
+Single attribute with tag (removed prefix 'hoop.'), without time, separated by SPACE and NOT to terminate by newline ('message' data will be terminated with newline).
+
+    <match hoop.**>
+      type hoop
+      hoop_server hoop-server.local:14000
+      path /hoop/log-%Y%m/log-%Y%m%d-%H.log
+      username hoopuser
+    
+      output_include_time false
+      output_include_tag true
+    
+      # If you want multiple attribute, specify like 'attr:field1,field2,field3'
+      output_data_type attr:message
+    
+      # field_separator allows 'SPACE', 'COMMA' and 'TAB'(default)
+      field_separator SPACE
+    
+      # add_newline 's default is true
+      add_newline false
+    
+      # tag 'hoop.foo.bar' is shrinked as 'foo.bar'
+      remove_prefix hoop
+    
+      # used for tags only remove_prefix string, like 'hoop'
+      default_tag unknown
+    </match>
+
+== TODO
+
+- consider what to do next
+- patches welcome!
+
+== Copyright
+
+Copyright:: Copyright (c) 2011- TAGOMORI Satoshi (tagomoris)
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "fluent-plugin-hoop"
+  gem.description = "Hoop (HDFS http-fs) plugin for Fluent event collector"
+  gem.homepage = "http://github.com/tagomoris/fluent-plugin-hoop"
+  gem.summary = gem.description
+  # gem.version = File.read("VERSION").strip
+  gem.authors = ["TAGOMORI Satoshi"]
+  gem.email = "tagomoris@gmail.com"
+  gem.has_rdoc = false
+  # gem.license = "Apache License v2.0"
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.add_dependency "fluentd", "~> 0.10.8"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  unless ENV['DEBUG']
+    ENV['FLUENT_TEST_DEBUG'] = 'TRUE'
+  end
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "fluent-plugin-hoop #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-hoop.gemspec

@@ -0,0 +1,75 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{fluent-plugin-hoop}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{TAGOMORI Satoshi}]
+  s.date = %q{2011-12-26}
+  s.description = %q{Hoop (HDFS http-fs) plugin for Fluent event collector}
+  s.email = %q{tagomoris@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".gitignore",
+    ".gitmodules",
+    "AUTHORS",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "fluent-plugin-hoop.gemspec",
+    "lib/fluent/plugin/out_hoop.rb",
+    "test/helper.rb",
+    "test/plugin/test_out_hoop.rb",
+    "test/plugin/test_out_hoop_realserver.rb",
+    "test/plugin/test_out_hoop_reconnect.rb"
+  ]
+  s.homepage = %q{http://github.com/tagomoris/fluent-plugin-hoop}
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.6}
+  s.summary = %q{Hoop (HDFS http-fs) plugin for Fluent event collector}
+  s.test_files = [%q{test/helper.rb}, %q{test/plugin/test_out_hoop.rb}, %q{test/plugin/test_out_hoop_realserver.rb}, %q{test/plugin/test_out_hoop_reconnect.rb}]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<rdoc>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.8"])
+      s.add_development_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_development_dependency(%q<simplecov>, [">= 0.5.4"])
+    else
+      s.add_dependency(%q<rdoc>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_dependency(%q<simplecov>, [">= 0.5.4"])
+    end
+  else
+    s.add_dependency(%q<rdoc>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
+    s.add_dependency(%q<simplecov>, [">= 0.5.4"])
+  end
+end
+

data/lib/fluent/plugin/out_hoop.rb

@@ -0,0 +1,339 @@
+module FluentExt; end
+module FluentExt::PlainTextFormatterMixin
+  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
+
+  attr_accessor :output_include_time, :output_include_tag, :output_data_type
+  attr_accessor :add_newline, :field_separator
+  attr_accessor :remove_prefix, :default_tag
+  
+  def configure(conf)
+    super
+
+    @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
+    @output_include_time = true if @output_include_time.nil?
+
+    @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
+    @output_include_tag = true if @output_include_tag.nil?
+
+    @output_data_type = conf['output_data_type']
+    @output_data_type = 'json' if @output_data_type.nil?
+
+    @field_separator = case @field_separator
+                       when 'SPACE' then ' '
+                       when 'COMMA' then ','
+                       else "\t"
+                       end
+    @add_newline = Fluent::Config.bool_value(conf['add_newline'])
+    if @add_newline.nil?
+      @add_newline = true
+    end
+
+    @remove_prefix = conf['remove_prefix']
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
+      @default_tag = conf['default_tag']
+      if @default_tag.nil? or @default_tag.length < 1
+        raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
+      end
+    end
+
+    # default timezone: utc
+    if conf['localtime'].nil? and conf['utc'].nil?
+      @utc = true
+      @localtime = false
+    elsif not @localtime and not @utc
+      @utc = true
+      @localtime = false
+    end
+    # mix-in default time formatter (or you can overwrite @timef on your own configure)
+    @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
+
+    @custom_attributes = []
+    if @output_data_type == 'json'
+      self.instance_eval {
+        def stringify_record(record)
+          record.to_json
+        end
+      }
+    elsif @output_data_type =~ /^attr:(.*)$/
+      @custom_attributes = $1.split(',')
+      if @custom_attributes.size > 1
+        self.instance_eval {
+          def stringify_record(record)
+            @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@field_separator)
+          end
+        }
+      elsif @custom_attributes.size == 1
+        self.instance_eval {
+          def stringify_record(record)
+            (record[@custom_attributes[0]] || 'NULL').to_s
+          end
+        }
+      else
+        raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
+      end
+    else
+      raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
+    end
+
+    if @output_include_time and @output_include_tag
+      if @add_newline and @remove_prefix
+        self.instance_eval {
+          def format(tag,time,record)
+            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+                tag == @remove_prefix
+              tag = tag[@removed_length..-1] || @default_tag
+            end
+            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
+          end
+        }
+      elsif @add_newline
+        self.instance_eval {
+          def format(tag,time,record)
+            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
+          end
+        }
+      elsif @remove_prefix
+        self.instance_eval {
+          def format(tag,time,record)
+            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+                tag == @remove_prefix
+              tag = tag[@removed_length..-1] || @default_tag
+            end
+            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
+          end
+        }
+      else
+        self.instance_eval {
+          def format(tag,time,record)
+            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
+          end
+        }
+      end
+    elsif @output_include_time
+      if @add_newline
+        self.instance_eval {
+          def format(tag,time,record);
+            @timef.format(time) + @field_separator + stringify_record(record) + "\n"
+          end
+        }
+      else
+        self.instance_eval {
+          def format(tag,time,record);
+            @timef.format(time) + @field_separator + stringify_record(record)
+          end
+        }
+      end
+    elsif @output_include_tag
+      if @add_newline and @remove_prefix
+        self.instance_eval {
+          def format(tag,time,record)
+            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+                tag == @remove_prefix
+              tag = tag[@removed_length..-1] || @default_tag
+            end
+            tag + @field_separator + stringify_record(record) + "\n"
+          end
+        }
+      elsif @add_newline
+        self.instance_eval {
+          def format(tag,time,record)
+            tag + @field_separator + stringify_record(record) + "\n"
+          end
+        }
+      elsif @remove_prefix
+        self.instance_eval {
+          def format(tag,time,record)
+            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+                tag == @remove_prefix
+              tag = tag[@removed_length..-1] || @default_tag
+            end
+            tag + @field_separator + stringify_record(record)
+          end
+        }
+      else
+        self.instance_eval {
+          def format(tag,time,record)
+            tag + @field_separator + stringify_record(record)
+          end
+        }
+      end
+    else # without time, tag
+      if @add_newline
+        self.instance_eval {
+          def format(tag,time,record);
+            stringify_record(record) + "\n"
+          end
+        }
+      else
+        self.instance_eval {
+          def format(tag,time,record);
+            stringify_record(record)
+          end
+        }
+      end
+    end
+  end
+
+  def stringify_record(record)
+    record.to_json
+  end
+
+  def format(tag, time, record)
+    if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
+      tag = tag[@removed_length..-1] || @default_tag
+    end
+    time_str = if @output_include_time
+                 @timef.format(time) + @field_separator
+               else
+                 ''
+               end
+    tag_str = if @output_include_tag
+                tag + @field_separator
+              else
+                ''
+              end
+    time_str + tag_str + stringify_record(record) + "\n"
+  end
+
+end
+
+class Fluent::HoopOutput < Fluent::TimeSlicedOutput
+  Fluent::Plugin.register_output('hoop', self)
+
+  config_set_default :buffer_type, 'memory'
+  config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
+  # config_param :tag_format, :string, :default => 'all' # or 'last'(last.part.of.tag => tag) or 'none'
+
+  config_param :hoop_server, :string   # host:port
+  config_param :path, :string          # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S and %T(tag, not-supported-yet)
+  config_param :username, :string      # hoop pseudo username
+  
+  include FluentExt::PlainTextFormatterMixin
+  config_set_default :output_include_time, true
+  config_set_default :output_include_tag, true
+  config_set_default :output_data_type, 'json'
+  config_set_default :field_separator, "\t"
+  config_set_default :add_newline, true
+  config_set_default :remove_prefix, nil
+
+  def initialize
+    super
+    require 'net/http'
+    require 'time'
+  end
+
+  def configure(conf)
+    if conf['path']
+      if conf['path'].index('%S')
+        conf['time_slice_format'] = '%Y%m%d%H%M%S'
+      elsif conf['path'].index('%M')
+        conf['time_slice_format'] = '%Y%m%d%H%M'
+      elsif conf['path'].index('%H')
+        conf['time_slice_format'] = '%Y%m%d%H'
+      end
+    end
+
+    super
+
+    unless /\A([a-zA-Z0-9][-a-zA-Z0-9.]*):(\d+)\Z/ =~ @hoop_server
+      raise Fluent::ConfigError, "Invalid config value on hoop_server: '#{@hoop_server}', needs SERVER_NAME:PORT"
+    end
+    @host = $1
+    @port = $2.to_i
+    unless @path.index('/') == 0
+      raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
+    end
+    @conn = nil
+    @header = {'Content-Type' => 'application/octet-stream'}
+
+    @f_separator = case @field_separator
+                   when 'SPACE' then ' '
+                   when 'COMMA' then ','
+                   else "\t"
+                   end
+  end
+
+  def start
+    super
+
+    # okey, net/http has reconnect feature. see test_out_hoop_reconnect.rb
+    conn = Net::HTTP.start(@host, @port)
+    begin
+      res = conn.request_get("/?op=status&user.name=#{@username}")
+      if res.code.to_i < 300 and res['Set-Cookie']
+        @authorized_header = {'Cookie' => res['Set-Cookie'].split(';')[0], 'Content-Type' => 'application/octet-stream'}
+      else
+        $log.error "initalize request failed, code: #{res.code}, message: #{res.body}"
+        raise Fluent::ConfigError, "initalize request failed, code: #{res.code}, message: #{res.body}"
+      end
+    rescue
+      $log.error "failed to connect hoop server: #{@host} port #{@port}"
+      raise
+    end
+    conn.finish
+    $log.info "connected hoop server: #{@host} port #{@port}"
+  end
+
+  def shutdown
+    super
+  end
+
+  def record_to_string(record)
+    record.to_json
+  end
+
+  def format(tag, time, record)
+    time_str = @timef.format(time)
+    time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
+  end
+
+  def path_format(chunk_key)
+    Time.strptime(chunk_key, @time_slice_format).strftime(@path)
+  end
+
+  def send_data(path, data, retries=0)
+    conn = Net::HTTP.start(@host, @port)
+    conn.read_timeout = 5
+    res = conn.request_put(path + "?op=append", data, @authorized_header)
+    if res.code == '401'
+      res = conn.request_get("/?op=status&user.name=#{@username}")
+      if res.code.to_i < 300 and res['Set-Cookie']
+        @authorized_header = {'Cookie' => res['Set-Cookie'].split(';')[0], 'Content-Type' => 'application/octet-stream'}
+      else
+        $log.error "Failed to update authorized cookie, code: #{res.code}, message: #{res.body}"
+        raise Fluent::ConfigError, "Failed to update authorized cookie, code: #{res.code}, message: #{res.body}"
+      end
+      res = conn.request_put(hdfs_path + "?op=append", data, @authorized_header)
+    end
+    if res.code == '404'
+      res = conn.request_post(path + "?op=create&overwrite=false", data, @authorized_header)
+    end
+    if res.code == '500'
+      if retries >= 3
+        raise StandardError, "failed to send_data with retry 3 times InternalServerError"
+      end
+      sleep 0.3 # yes, this is a magic number
+      res = send_data(path, data, retries + 1)
+    end
+    conn.finish
+    if res.code != '200' and res.code != '201'
+      $log.warn "failed to write data to path: #{path}, code: #{res.code} #{res.message}"
+    end
+    res
+  end
+
+  def write(chunk)
+    hdfs_path = path_format(chunk.key)
+    begin
+      send_data(hdfs_path, chunk.read)
+    rescue
+      $log.error "failed to communicate server, #{@host} port #{@port}, path: #{hdfs_path}"
+      raise
+    end
+    hdfs_path
+  end
+end