fluentd 0.10.0

data/AUTHORS

@@ -0,0 +1 @@
+FURUHASHI Sadayuki <frsyuki _at_ gmail.com>

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2011 FURUHASHI Sadayuki
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/ChangeLog

@@ -0,0 +1,178 @@
+
+Release 0.10.0 - 2011/10/16
+
+  * Removed Event class
+    * def format(tag, event) -> format(time, tag, record)
+	* def emit(tag, event) -> emit(tag, time, record)
+  * Added plugin APIs
+    * Added Configurable
+    * Added TimeFormatter
+    * Added SetTimeKeyMixin
+    * Added SetTagKeyMixin
+    * Added InputTestDriver
+  * Added Gemfile, gemspec and .rvmrc files
+
+
+Release 0.9.20 - 2011/10/07
+
+  * Config#has_key? sets used flag not to show warning
+  * in_tcp supports JSON
+  * Depends on Yajl
+  * Fixed fluentd -r option
+  * Fixed autogen.sh to create plugins directory
+
+
+Release 0.9.19 - 2011/10/07
+
+  * Secondary outputs work only when error count exceededs retry_limit
+  * Secondary outputs retry upto secondary_limit
+  * Added Output#secondary_init
+  * Added unit test driver for output plugins
+  * Added Fluent::Test.test? method
+  * Added comparision methods (<=>, ==, eql? and hash) to Event class
+  * Added test cases for out_copy, out_roundrobin and out_file
+
+
+Release 0.9.18 - 2011/10/04
+
+  * Performance improvements
+    * in_tcp uses lazy deserialization
+	* Engine caches results of Match#match
+	* out_file caches results of strftime
+  * Fixed TextParser.register_template
+
+
+Release 0.9.17 - 2011/10/03
+
+  * TimeSlicedOutput caches results of strftime() for performance
+  * Fixed TextParser.register_template (thanks eiichiroi)
+  * Fixed fluent-cat --port argument to work
+
+
+Release 0.9.16 - 2011/10/01
+
+  * Fixed it can't use relative path for --plugin command line argument
+  * Fixed out_copy to work with non-repeatable event streams
+
+
+Release 0.9.15 - 2011/09/21
+
+  * Set default encoding to ASCII-8BIT
+  * Fixed thread issues in BasicBuffer
+  * Performance improvements
+
+
+Release 0.9.14 - 2011/09/21
+
+  * Fixed to write unexpected error to log files
+  * Plugin loader uses GemSpec if it is available to load newer plugin
+  * Added Config.bool_value method for plugins
+
+
+Release 0.9.13 - 2011/09/21
+
+  * Updated out_file plugin
+    * Uses localtime by default
+    * Uses \t to separate time, tag and JSON.
+    * Uses ISO-8601 for the time format
+
+
+Release 0.9.12 - 2011/09/20
+
+  * Updated out_file plugin
+    * Supports log compression
+	* Slices log files every day by default
+
+
+Release 0.9.11 - 2011/09/19
+
+  * Added out_null plugin
+  * Added out_roundrobin plugin
+  * in_http supports keep-alive
+  * in_tcp and in_unix detect network failure
+
+
+Release 0.9.10 - 2011/09/07
+
+  * Periodic timer enqueues buffer chunks only when queue is empty
+
+
+Release 0.9.9 - 2011/09/06
+
+  * in_tail strips \n at the end of log line
+  * Uses newer one if multiple plugins are found
+
+
+Release 0.9.8 - 2011/08/16
+
+  * in_http plugin supports 'Expect: 100-continue' header
+  * Added --user and --group command line arguments to change privilege
+  * Changed default retry_limit parameter from 8 to 17
+  * Changed default buffer_chunk_limit parameter from 1m to 16m
+  * Changed default buffer_queue_limit parameter from 100 to 64
+
+
+Release 0.9.7 - 2011/08/06
+
+  * Load plugins named fluent_plugin.rb installed by RubyGems
+  * Load plugins located on fluent/plugin/{buf,in,out}_{type} installed by RubyGems
+  * Removed out_tdlog plugin to fluent-plugin-td gem
+
+
+Release 0.9.6 - 2011/08/06
+
+  * Force flush buffered events on SIGUSR1
+  * out_tdlog uses event tag for the database and table name
+  * Fluent can log Fluent logs
+  * Colorize Fluent logs on console
+  * Changed default unix domain socket path to /var/run/fluent/fluent.sock
+  * Makefile.am and configure.in aware fakeroot
+
+
+Release 0.9.5 - 2011/07/23
+
+  * Added <secondary> tag that fallbacks to secondary output when it can't
+    write to primary output.
+  * <match> pattern supports **; matches any fragments recursively.
+  * <match> pattern supports {x,y,...}; matches any of the inner patterns.
+
+
+Release 0.9.4 - 2011/07/22
+
+  * Switched EventMachine to Cool.io
+  * in_tail continues to follow file if it is deleted on Linux
+  * Added fluentd --setup option that installs sample configuration file
+  * Added out_tdlog plugin
+
+
+Release 0.9.3 - 2011/07/05
+
+  * Disable EventMachine.epoll because it doesn't work correctly on
+    Ruby 1.9 with threads
+  * in_tcp and in_unix uses EventMachine instead of threads
+  * Set BasicSocket.do_not_reverse_lookup = true
+
+
+Release 0.9.2 - 2011/07/03
+
+  * Added TimeSlicedOutput that splits buffer chunk exactly based on the time
+  * Changed structure of buffers from queue to map + queue
+  * Show waring if configuration parameters are not used
+  * Added out_time_file plugin
+
+
+Release 0.9.1 - 2011/06/26
+
+  * Added in_tail plugin
+  * Added in_syslog plugin
+  * Added out_stdout plugin
+  * Added out_tcp plugin
+  * Added out_unix plugin
+  * Improved performance of in_http plugin by replacing WEBrick + thread
+     with EventMachine
+
+
+Release 0.9.0 - 2011/06/20
+
+  * First release
+

data/README.rdoc

@@ -0,0 +1,57 @@
+= Fluent
+
+Fluent is an event collector system. It's said that Fluent is generalized version of syslogd, which can deal with JSON object for the log message.
+
+== Archtecture
+
+Fluent collects events from various data sources and write them to files, database or other storages:
+
+    
+    Web apps  ---+                 +--> file
+                 |                 |
+                 +-->           ---+
+    /var/log  ------>  fluentd  ------> mail
+                 +-->           ---+
+                 |                 |
+    apache    ----                 +--> fluentd
+    
+
+Fluent also supports log transfer:
+
+    
+    Web server
+    +---------+
+    | fluentd -------
+    +---------+|     |
+     +---------+     |
+                     |
+    Proxy server     |    Log server, Amazon S3, HDFS, ...
+    +---------+      +--> +---------+
+    | fluentd ----------> | fluentd ||
+    +---------+|     +--> +---------+|
+     +---------+     |     +---------+
+                     |
+    Database server  |
+    +---------+      |
+    | fluentd ---------> mail
+    +---------+|
+     +---------+
+    
+
+An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time when the event occurs. Record is a JSON object.
+
+
+== Quick Start
+
+  $ gem install fluentd
+  $ fluentd &
+  $ echo '{"json":"message"}' | fluent-cat debug.test
+
+
+Web site::  http://fluentd.org/
+Documents:: http://fluentd.org/doc/
+Source repository:: http://github.com/fluent
+Author:: Sadayuki Furuhashi
+Copyright:: (c) 2011 FURUHASHI Sadayuki
+License:: Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,62 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "fluentd"
+    gemspec.summary = "Fluent event collector"
+    gemspec.author = "Sadayuki Furuhashi"
+    gemspec.email = "frsyuki@gmail.com"
+    gemspec.homepage = "http://fluentd.org/"
+    gemspec.has_rdoc = false
+    gemspec.require_paths = ["lib"]
+    gemspec.add_dependency "msgpack", "~> 0.4.4"
+    gemspec.add_dependency "json", ">= 1.4.3"
+    gemspec.add_dependency "yajl-ruby", "~> 1.0.0"
+    gemspec.add_dependency "cool.io", "~> 1.0.0"
+    gemspec.add_dependency "http_parser.rb", "~> 0.5.1"
+    gemspec.add_dependency "rake", ">= 0.9.2"
+    gemspec.test_files = Dir["test/**/*.rb"]
+    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
+      %w[fluent.conf VERSION AUTHORS Rakefile COPYING fluentd.gemspec]
+    gemspec.executables = ['fluentd', 'fluent-cat', 'fluent-gem']
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/*_test.rb']
+  t.ruby_opts = ['-rubygems'] if defined? Gem
+  t.ruby_opts << '-I.'
+end
+
+VERSION_FILE = "lib/fluent/version.rb"
+
+file VERSION_FILE => ["VERSION"] do |t|
+  version = File.read("VERSION").strip
+  File.open(VERSION_FILE, "w") {|f|
+    f.write <<EOF
+module Fluent
+
+VERSION = '#{version}'
+
+end
+EOF
+  }
+end
+
+task :test => [:base_test]
+
+Rake::TestTask.new(:base_test) do |t|
+  t.libs << "test"
+  t.test_files = (Dir["test/*.rb"] + Dir["test/plugin/*.rb"] - ["helper.rb"]).sort
+  t.verbose = true
+  #t.warning = true
+end
+
+task :default => [VERSION_FILE, :build, :gemspec]
+

data/VERSION

@@ -0,0 +1 @@
+0.10.0

data/bin/fluent-cat

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/cat'

data/bin/fluent-gem

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+require 'rubygems/gem_runner'
+require 'rubygems/exceptions'
+begin
+  Gem::GemRunner.new.run ARGV
+rescue Gem::SystemExitException => e
+  exit e.exit_code
+end

data/bin/fluentd

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/fluentd'

data/fluent.conf

@@ -0,0 +1,78 @@
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  type tcp
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  type http
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+## match tag=system.** and forward to another fluent server
+#<match system.**>
+#  type tcp
+#  host 192.168.0.11
+#  <secondary>
+#    host 192.168.0.12
+#  </secondary>
+#</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  type copy
+#  <store>
+#    type tcp
+#    host 192.168.0.13
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#  </store>
+#  <store>
+#    type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+

data/fluentd.gemspec

@@ -0,0 +1,116 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{fluentd}
+  s.version = "0.10.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Sadayuki Furuhashi"]
+  s.date = %q{2011-10-16}
+  s.email = %q{frsyuki@gmail.com}
+  s.executables = ["fluentd", "fluent-cat", "fluent-gem"]
+  s.extra_rdoc_files = [
+    "ChangeLog",
+     "README.rdoc"
+  ]
+  s.files = [
+    "AUTHORS",
+     "COPYING",
+     "Rakefile",
+     "VERSION",
+     "bin/fluent-cat",
+     "bin/fluent-gem",
+     "bin/fluentd",
+     "fluent.conf",
+     "fluentd.gemspec",
+     "lib/fluent/buffer.rb",
+     "lib/fluent/command/cat.rb",
+     "lib/fluent/command/fluentd.rb",
+     "lib/fluent/config.rb",
+     "lib/fluent/engine.rb",
+     "lib/fluent/env.rb",
+     "lib/fluent/event.rb",
+     "lib/fluent/input.rb",
+     "lib/fluent/load.rb",
+     "lib/fluent/log.rb",
+     "lib/fluent/match.rb",
+     "lib/fluent/mixin.rb",
+     "lib/fluent/output.rb",
+     "lib/fluent/parser.rb",
+     "lib/fluent/plugin.rb",
+     "lib/fluent/plugin/buf_file.rb",
+     "lib/fluent/plugin/buf_memory.rb",
+     "lib/fluent/plugin/buf_zfile.rb",
+     "lib/fluent/plugin/in_http.rb",
+     "lib/fluent/plugin/in_stream.rb",
+     "lib/fluent/plugin/in_syslog.rb",
+     "lib/fluent/plugin/in_tail.rb",
+     "lib/fluent/plugin/out_copy.rb",
+     "lib/fluent/plugin/out_file.rb",
+     "lib/fluent/plugin/out_null.rb",
+     "lib/fluent/plugin/out_roundrobin.rb",
+     "lib/fluent/plugin/out_stdout.rb",
+     "lib/fluent/plugin/out_stream.rb",
+     "lib/fluent/plugin/out_test.rb",
+     "lib/fluent/test.rb",
+     "lib/fluent/test/base.rb",
+     "lib/fluent/test/input_test.rb",
+     "lib/fluent/test/output_test.rb",
+     "lib/fluent/version.rb",
+     "test/helper.rb",
+     "test/match.rb",
+     "test/plugin/in_http.rb",
+     "test/plugin/in_stream.rb",
+     "test/plugin/out_copy.rb",
+     "test/plugin/out_file.rb",
+     "test/plugin/out_roundrobin.rb",
+     "test/plugin/out_stream.rb"
+  ]
+  s.homepage = %q{http://fluentd.org/}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Fluent event collector}
+  s.test_files = [
+    "test/helper.rb",
+     "test/match.rb",
+     "test/plugin/in_http.rb",
+     "test/plugin/in_stream.rb",
+     "test/plugin/out_copy.rb",
+     "test/plugin/out_file.rb",
+     "test/plugin/out_roundrobin.rb",
+     "test/plugin/out_stream.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<msgpack>, ["~> 0.4.4"])
+      s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
+      s.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+      s.add_runtime_dependency(%q<cool.io>, ["~> 1.0.0"])
+      s.add_runtime_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+      s.add_runtime_dependency(%q<rake>, [">= 0.9.2"])
+    else
+      s.add_dependency(%q<msgpack>, ["~> 0.4.4"])
+      s.add_dependency(%q<json>, [">= 1.4.3"])
+      s.add_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+      s.add_dependency(%q<cool.io>, ["~> 1.0.0"])
+      s.add_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
+    end
+  else
+    s.add_dependency(%q<msgpack>, ["~> 0.4.4"])
+    s.add_dependency(%q<json>, [">= 1.4.3"])
+    s.add_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+    s.add_dependency(%q<cool.io>, ["~> 1.0.0"])
+    s.add_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
+  end
+end
+