fluentd 0.12.0.pre.1 → 0.12.0.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38b8348d98cec8b31c27e634bd22a9afee293ad0
-  data.tar.gz: 983f2d287d63a26637c95266ce4cd1a85dcd1757
+  metadata.gz: 5728345e3e45be5f8327ea9cfe0268b958c78d92
+  data.tar.gz: 453abbcdf2c21a2ff97fb6da4b9e4468c0519de1
 SHA512:
-  metadata.gz: 0b62fb35a7a546cfa741564854091983fdf1d05ff7a85c44cae4dab1889b9f5eb3c5940b32e5739e05bd43a8c7fdbce05723d76a42959fb04cd7b9a6090aee6b
-  data.tar.gz: 8538fff4fcff3ca9fd72ac88681490a7d9e263d53ea51d410b82e486cf6896df7857d158fb6f5dceb55f5ef78072d2bfc3b3bf30faf3dbd00be1b7603452c761
+  metadata.gz: 0c2878ba5c1d0d45e12d75eea0ed5700ec513b2dde96ecbf8efce4d6e2a0d81eeb9b4a20b7d43b3a053c9c84c24caf3a794fdc1af750b37d7d8e4e3feaf4bdcd
+  data.tar.gz: 68f2edd339b18575244d1643df2144f71b4dc2d48dac9355071fbc4a0ceb141bddcdee67189d84d1bf451b8445b83fe50bc545ecc6c866593012702c57b1bb24

data/.gitignore

@@ -17,5 +17,5 @@ fluent-gem
 fluentd
 pkg/*
 test/tmp/*
-spec/config/tmp/*
+test/config/tmp/*
 make_dist.sh

data/.travis.yml

@@ -14,6 +14,7 @@ os:
 branches:
   only:
     - master
+    - v0.10
 
 gemfile:
   - Gemfile

data/ChangeLog

@@ -1,3 +1,24 @@
+Release 0.10.55 - 2014/10/16
+
+* config: Change v1's non-quoted string literal behaviour to v0's behaviour
+  https://github.com/fluent/fluentd/pull/436
+* config: Relax '@' handling in the plugin configuration for backward compatibility
+  https://github.com/fluent/fluentd/pull/455
+* config: Add --use-v0-config for keeping old configuration
+  https://github.com/fluent/fluentd/pull/434
+* config: Add single quoted literal in v1 configuration
+  https://github.com/fluent/fluentd/pull/437
+* config: Improve error message of Array / Hash parse error
+  https://github.com/fluent/fluentd/pull/457
+* input: Reduce shutdown time in some network related plugins when use Cool.io 1.2
+  https://github.com/fluent/fluentd/pull/440
+* parser: Use ParserError instead of general exception classes
+  https://github.com/fluent/fluentd/pull/453
+* Remove unused zfile buffer plugin
+  https://github.com/fluent/fluentd/pull/445
+
+Release 0.10.54 - skipped
+
 Release 0.10.53 - 2014/08/21
 
 * in_tail: Fix forget to detach Closer timer object

data/README.md

@@ -1,9 +1,8 @@
-Fluentd: Open-Source Data Collector
+Fluentd: Open-Source Log Collector
 ===================================
 
 [<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 
-
 [Fluentd](http://fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](http://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
@@ -12,6 +11,13 @@ Fluentd: Open-Source Data Collector
 
 An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
 
+## Example Use Cases
+
+Use Case | Description | Diagram
+-------- | ------------|:---------:
+Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/assets/img/recipes/elasticsearch-s3-fluentd.png" height="150"/>
+Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
+Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/assets/img/datasources/asynchronous_logging.png" height="150"/>
 
 ## Quick Start
 
@@ -24,6 +30,8 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
 
+<p align="center"><img width="500" src="http://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
+
 ## More Information
 
 - Website: http://fluentd.org/

data/Rakefile

@@ -4,10 +4,8 @@ require "bundler/gem_tasks"
 require 'fileutils'
 require 'rake/testtask'
 require 'rake/clean'
-require 'rspec/core'
-require 'rspec/core/rake_task'
 
-task :test => [:base_test, :spec]
+task :test => [:base_test]
 
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
@@ -15,7 +13,7 @@ Rake::TestTask.new(:base_test) do |t|
   #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
   #  $ bundle exec rake base_test TEST=test/test_*.rb
   t.libs << "test"
-  t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
+  t.test_files = Dir["test/**/test_*.rb"].sort
   t.verbose = true
   #t.warning = true
 end
@@ -26,17 +24,10 @@ task :parallel_test do
   FileUtils.rm_rf('./test/tmp')
 end
 
-desc 'Run rspec based test'
-RSpec::Core::RakeTask.new(:spec) do |t|
-  t.rspec_opts = %w[-c -f progress -r ./spec/spec_helper.rb]
-  t.pattern = 'spec/**/*_spec.rb'
-  t.verbose = true
-end
-
-desc 'Run rspec with simplecov'
+desc 'Run test with simplecov'
 task :coverage do |t|
   ENV['SIMPLE_COV'] = '1'
-  Rake::Task["spec"].invoke
+  Rake::Task["test"].invoke
 end
 
 task :default => [:test, :build]

data/example/v1_literal_example.conf

@@ -0,0 +1,36 @@
+<section1>
+  key1  'text'     # text
+  key2  '\''       # ' (1 char)
+  key3  '\\'       # \ (1 char)
+  key4  '\t'       # \t (2 char)
+  key5  '\['       # \[ (2 char)
+  key6  '\\['      # \[ (2 char)
+  key7  '#t'       # #t (2 char)
+  key8  '\#{test}' # \#{test} (8 char)
+  key9  '#{test}'  # #{test} (7 char)
+  key10 '\[(?<time>[^\]]*)\] (?<message>.*)'    # \[(?<time>[^\]]*\] (?<message>.*)
+</section1>
+<section2>
+  key1  "text"     # text
+  key2  "\""       # " (1 char)
+  key3  "\\"       # \ (1 char)
+  key4  "\t"       # TAB (1 char)
+  key5  "\["       # [ (1 char)
+  key6  "\\["      # \[ (2 char)
+  key7  "#t"       # #t (2 char)
+  key8  "\#{test}" # #{test} (7 char)
+  key9  "#{test}"  # replaced by eval('test')
+  key10 "\\[(?<time>[^\\]]*)\\] (?<message>.*)" # \[(?<time>[^\]]*\] (?<message>.*)
+</section2>
+<section3>
+  key1  text     # text
+  key2  \        # \ (1 char)
+  key3  \\       # \\ (2 char)
+  key4  \t       # \t (2 char)
+  key5  \[       # \[ (2 char)
+  key6  \\[      # \\[ (3 char)
+  key7  #t       # #t (2 char)
+  key8  \#{test} # \#{test} (8 char)
+  key9  #{test}  # #{test} (7 char)
+  key10 \[(?<time>[^\]]*)\] (?<message>.*)    # \[(?<time>[^\]]*\] (?<message>.*)
+</section3>

data/fluentd.gemspec

@@ -24,12 +24,15 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
+  gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
+  gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
+  gem.add_runtime_dependency("string-scrub", [">= 0.0.3"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock")
   gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("rspec", ["~> 3.0.0"])
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
+  gem.add_development_dependency("test-unit", ["~> 3.0.2"])
 end

data/lib/fluent/buffer.rb

@@ -44,20 +44,25 @@ module Fluent
     def before_shutdown(out)
     end
 
-    #def emit(key, data, chain)
-    #end
+    def emit(key, data, chain)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def keys
-    #end
+    def keys
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def push(key)
-    #end
+    def push(key)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def pop(out)
-    #end
+    def pop(out)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def clear!
-    #end
+    def clear!
+      raise NotImplementedError, "Implement this method in child class"
+    end
   end
 
 
@@ -71,27 +76,33 @@ module Fluent
 
     attr_reader :key
 
-    #def <<(data)
-    #end
+    def <<(data)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def size
-    #end
+    def size
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
     def empty?
       size == 0
     end
 
-    #def close
-    #end
+    def close
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def purge
-    #end
+    def purge
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def read
-    #end
+    def read
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def open
-    #end
+    def open
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
     def write_to(io)
       open {|i|
@@ -116,6 +127,8 @@ module Fluent
 
     def initialize
       super
+      @map = nil # chunks to store data
+      @queue = nil # chunks to be flushed
       @parallel_pop = true
     end
 
@@ -164,18 +177,14 @@ module Fluent
       key = key.to_s
 
       synchronize do
-        top = (@map[key] ||= new_chunk(key))  # TODO generate unique chunk id
+        # chunk unique id is generated in #new_chunk
+        chunk = (@map[key] ||= new_chunk(key))
 
-        if storable?(top, data)
+        if storable?(chunk, data)
           chain.next
-          top << data
+          chunk << data
           return false
 
-          ## FIXME
-          #elsif data.bytesize > @buffer_chunk_limit
-          #  # TODO
-          #  raise BufferChunkLimitError, "received data too large"
-
         elsif @queue.size >= @buffer_queue_limit
           raise BufferQueueLimitError, "queue size exceeds limit"
         end
@@ -185,9 +194,12 @@ module Fluent
           $log.warn "This may occur problems in the output plugins ``at this server.``"
           $log.warn "To avoid problems, set a smaller number to the buffer_chunk_limit"
           $log.warn "in the forward output ``at the log forwarding server.``"
+          ### TODO
+          # raise BufferChunkLimitError, "received data too large"
         end
 
-        nc = new_chunk(key)  # TODO generate unique chunk id
+        # chunk unique id is generated in #new_chunk
+        nc = new_chunk(key)
         ok = false
 
         begin
@@ -196,13 +208,17 @@ module Fluent
 
           flush_trigger = false
           @queue.synchronize {
-            enqueue(top)
+            enqueue(chunk) # this is buffer enqueue *hook*
             flush_trigger = @queue.empty?
-            @queue << top
+            @queue << chunk # actual enqueue
             @map[key] = nc
           }
 
           ok = true
+          # false: queue have 1 or more chunks before this emit
+          #        so this enqueue is not a trigger to flush
+          # true: queue have no chunks before this emit
+          #       so this enqueue is a trigger to flush this buffer ASAP
           return flush_trigger
         ensure
           nc.purge unless ok
@@ -230,25 +246,31 @@ module Fluent
       total
     end
 
-    #def new_chunk(key)
-    #end
+    def new_chunk(key)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def resume
-    #end
+    def resume
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
-    #def enqueue(chunk)
-    #end
+    # enqueueing is done by #push
+    # this method is actually 'enqueue_hook'
+    def enqueue(chunk)
+      raise NotImplementedError, "Implement this method in child class"
+    end
 
+    # get the chunk specified by key, and push it into queue
     def push(key)
       synchronize do
-        top = @map[key]
-        if !top || top.empty?
+        chunk = @map[key]
+        if !chunk || chunk.empty?
           return false
         end
 
         @queue.synchronize do
-          enqueue(top)
-          @queue << top
+          enqueue(chunk)
+          @queue << chunk
           @map.delete(key)
         end
 
@@ -256,6 +278,8 @@ module Fluent
       end  # synchronize
     end
 
+    # shift a chunk from queue, write and purge it
+    # returns boolean to indicate whether this buffer have more chunk to be flushed or not
     def pop(out)
       chunk = nil
       @queue.synchronize do
@@ -270,21 +294,24 @@ module Fluent
       end
 
       begin
+        # #push(key) does not push empty chunks into queue.
+        # so this check is nonsense...
         if !chunk.empty?
           write_chunk(chunk, out)
         end
 
-        empty = false
+        queue_empty = false
         @queue.synchronize do
           @queue.delete_if {|c|
             c.object_id == chunk.object_id
           }
-          empty = @queue.empty?
+          queue_empty = @queue.empty?
         end
 
         chunk.purge
 
-        return !empty
+        # return to be flushed once more immediately, or not
+        return !queue_empty
       ensure
         chunk.mon_exit
       end

data/lib/fluent/command/fluentd.rb

@@ -70,7 +70,8 @@ op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit er
   opts[:suppress_interval] = s.to_i
 }
 
-op.on('--suppress-repeated-stacktrace', "suppress repeated stacktrace", TrueClass) {|b|
+op.on('--suppress-repeated-stacktrace [VALUE]', "suppress repeated stacktrace", TrueClass) {|b|
+  b = true if b.nil?
   opts[:suppress_repeated_stacktrace] = b
 }
 
@@ -78,10 +79,14 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
   opts[:without_source] = b
 }
 
-op.on('--use-v1-config', "Use v1 configuration format", TrueClass) {|b|
+op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
   opts[:use_v1_config] = b
 }
 
+op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
+  opts[:use_v1_config] = !b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/config/basic_parser.rb

@@ -27,6 +27,7 @@ module Fluent
 
       LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
+      SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
 
       module ClassMethods
         def symbol(string)
@@ -78,6 +79,10 @@ module Fluent
         skip(SPACING)
       end
 
+      def spacing_without_comment
+        skip(SPACING_WITHOUT_COMMENT)
+      end
+
       def parse_error!(message)
         raise ConfigParseError, "#{message} at #{error_sample}"
       end