fluent-plugin-cat-sweep 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d592c2a70cecbfcd6e6d9d034d7f8587d98c91d
-  data.tar.gz: 658b34062c8f638cdf26e63a81107a56fac2710c
+  metadata.gz: 7c416b3c5df83179c396c4343661ca0d6edabf9c
+  data.tar.gz: 99f032ae97d3276572f4d83c986a2b8cf9ed47bd
 SHA512:
-  metadata.gz: d3ba2f13b2cfdddd6146de2ea4d7ed8887f01fcce256be62f657e70a70c708dcc13e3acec9cdcae18d42d02e58a24aa3828a0b144fb54fb00fe207c1429be33a
-  data.tar.gz: 17f4d69c1cba812a3fb49c5313e875209deb554cacb66f1693ca7849e9b89f53a72e23c5ede43143b069c5218c78a2cc18f038f3d9b30fecd2041ac2242f4239
+  metadata.gz: 7e04a78004b577267659870f282bca66eb6212892a013db0f2c1ada39d1c7b4829264afe1c2183c499e9b3a339f5da179fce52162119c8a5f17e564c5b595873
+  data.tar.gz: 29e6a7cc393cb249f9008c387f4b7d42740177b6c1908aecbce54b31b8181ca910bae46a1d282eea19707e9bb01cb1aff802984316e079988812b46061ea92f0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.2
+
+Enhancements:
+
+* Add file_event_stream option
+* Speedup test
+
 # 0.1.1
 
 Enhancements:

data/README.md

@@ -81,6 +81,11 @@ Our assumption is that this mechanism should provide more durability than `in_ta
 
   # Optional. default 5 seconds.
   run_interval 10
+
+  # Optional. Emit entire file contents as an event, default emits each line as an event.
+  # This assures that fluentd emits the entire file contents together. Please note that buffer_chunk_limit
+  # must be larger than bytes in a file to be sent by buffered output plugins such as out_forward, out_s3.
+  file_event_stream false
 </source>
 ```
 

data/fluent-plugin-cat-sweep.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cat-sweep"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["Civitaspo(takahiro.nakayama)", "Naotoshi Seo"]
   spec.email         = ["civitaspo@gmail.com", "sonots@gmail.com"]
 

data/lib/fluent/plugin/in_cat_sweep.rb

@@ -19,8 +19,8 @@ module Fluent
     config_param :oneline_max_bytes,       :integer, :default => 536870912 # 512MB
     config_param :move_to,                 :string,  :default => '/tmp'
     config_param :remove_after_processing, :bool,    :default => false
-    config_param :run_interval,            :integer, :default => 5
-
+    config_param :run_interval,            :time,    :default => 5
+    config_param :file_event_stream,       :bool,    :default => false
 
     # To support log_level option implemented by Fluentd v0.10.43
     unless method_defined?(:log)
@@ -184,22 +184,52 @@ module Fluent
       buffer_clean!
     end
 
-    def emit_message(message)
-      if message
-        @parser.parse(message) do |time, record|
-          unless time and record
-            raise FormatError,
-              "in_cat_sweep: pattern not match: #{message.inspect}"
-          end
+    def emit_line(line)
+      if line
+        time, record = parse_line(line)
+        if time and record
           router.emit(@tag, time, record)
         end
       end
     end
 
+    def emit_file(fp)
+      entries = []
+      read_each_line(fp) do |line|
+        if line
+          entry = parse_line(line)
+          entries << entry if entry
+        end
+      end
+      unless entries.empty?
+        es = ArrayEventStream.new(entries)
+        router.emit_stream(@tag, es)
+      end
+    end
+
+    def parse_line(line)
+      entry = nil
+      @parser.parse(line) do |time, record|
+        if time && record
+          entry = [time, record]
+        else
+          # We want to fail an entire file on `pattern not match`
+          # This behavior makes it easy to recover with manual fix operation
+          raise FormatError,
+            "in_cat_sweep: pattern not match: #{line.inspect}"
+        end
+      end
+      entry
+    end
+
     def process(original_filename, processing_filename)
       File.open(processing_filename, 'r') do |tfile|
-        read_each_line(tfile) do |line|
-          emit_message(line)
+        if @file_event_stream
+          emit_file(tfile)
+        else
+          read_each_line(tfile) do |line|
+            emit_line(line)
+          end
         end
         log.debug { %[in_cat_sweep: process: {filename:"#{original_filename}",size:#{tfile.size}}] }
       end

data/test/helper.rb

@@ -6,3 +6,23 @@ unless defined?(Test::Unit::AssertionFailedError)
   class Test::Unit::AssertionFailedError < StandardError
   end
 end
+
+# Reduce sleep period at
+# https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
+module Fluent
+  module Test
+    class TestDriver
+      def run(num_waits = 10, &block)
+        @instance.start
+        begin
+          # wait until thread starts
+          # num_waits.times { sleep 0.05 }
+          sleep 0.05
+          return yield
+        ensure
+          @instance.shutdown
+        end
+      end
+    end
+  end
+end

data/test/test_in_cat_sweep.rb

@@ -19,6 +19,12 @@ class CatSweepInputTest < Test::Unit::TestCase
 
   CONFIG_BASE = %[
     file_path_with_glob #{TMP_DIR_FROM}/*
+    run_interval 0.05
+  ]
+
+  CONFIG_MINIMUM_REQUIRED = CONFIG_BASE + %[
+    format tsv
+    waiting_seconds 5
   ]
 
   def create_driver(conf, use_v1 = true)
@@ -38,19 +44,24 @@ class CatSweepInputTest < Test::Unit::TestCase
       d = create_driver(CONFIG_BASE + %[format tsv])
     end
 
-    d = create_driver(CONFIG_BASE + %[
-      format tsv
-      waiting_seconds 5
-      ])
+    d = create_driver(CONFIG_MINIMUM_REQUIRED)
 
     assert_equal "#{TMP_DIR_FROM}/*", d.instance.instance_variable_get(:@file_path_with_glob)
     assert_equal 'tsv', d.instance.instance_variable_get(:@format)
     assert_equal 5, d.instance.instance_variable_get(:@waiting_seconds)
   end
 
+  def test_configure_file_event_stream
+    d = create_driver(CONFIG_MINIMUM_REQUIRED)
+    assert { false == d.instance.file_event_stream }
+
+    d = create_driver(CONFIG_MINIMUM_REQUIRED + %[file_event_stream true])
+    assert { true == d.instance.file_event_stream }
+  end
+
   def compare_test_result(emits, tests)
     emits.each_index do |i|
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert { tests[i]['expected'] == emits[i][2]['message'] }
     end
   end
 
@@ -70,24 +81,27 @@ class CatSweepInputTest < Test::Unit::TestCase
       ]
     }
 
-  TEST_CASES.each do |format, test_cases|
-    test_case_name = "test_msg_process_#{format}"
-    define_method(test_case_name) do
-      File.open("#{TMP_DIR_FROM}/#{test_case_name}", 'w') do |io|
-        test_cases.each do |test|
-          io.write(test['msg'])
+  [false, true].each do |file_event_stream|
+    TEST_CASES.each do |format, test_cases|
+      test_case_name = "test_msg_process_#{format}_file_event_stream_#{file_event_stream}"
+      define_method(test_case_name) do
+        File.open("#{TMP_DIR_FROM}/#{test_case_name}", 'w') do |io|
+          test_cases.each do |test|
+            io.write(test['msg'])
+          end
         end
-      end
 
-      d = create_driver(CONFIG_BASE + %[
-        format #{format}
-        waiting_seconds 0
-        keys hdfs_path,unixtimestamp,label,message
-        ])
-      d.run
+        d = create_driver(CONFIG_BASE + %[
+          format #{format}
+          file_event_stream #{file_event_stream}
+          waiting_seconds 0
+          keys hdfs_path,unixtimestamp,label,message
+          ])
+        d.run
 
-      compare_test_result(d.emits, test_cases)
-      assert(Dir.glob("#{TMP_DIR_FROM}/#{test_case_name}*").empty?)
+        compare_test_result(d.emits, test_cases)
+        assert { Dir.glob("#{TMP_DIR_FROM}/#{test_case_name}*").empty? }
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cat-sweep
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Civitaspo(takahiro.nakayama)
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-30 00:00:00.000000000 Z
+date: 2016-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to cat files and move them.