tailf2kafka 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a5efc928874d8b1900268152cf26bc6b6382491
-  data.tar.gz: 8814ee2e23060c01064ae53e468c7514e9ec0fb4
+  metadata.gz: 517eb3cc0c07e25383e56e63d10a09a12c2b5835
+  data.tar.gz: 69136fb7321b932d4aee23368f1b8b363ebc9fa2
 SHA512:
-  metadata.gz: bc6f6ff3b03436364582f1279f62aa9862b61a8ae9c7e0217ed420cec647ee3b284031ef6d19c42c0ecae878e886cfc68ee15c29ce9c8a7dee773b232efd80c2
-  data.tar.gz: b145745c6f763d760827e85050722f8a706794f64dc86b005b6040f6470bd95a9abd515a45938ab867f883fce0cfb0bf0cc98bac652049787e1ba76a8215a07b
+  metadata.gz: e82b708355566b9902c765fbac0c811e4fbc3695af51afe86cc5a617ba3cbdbc58e50c73769fbabf687b51e99d8a686d1ab9f768bb35444c11d1a82e7775ca98
+  data.tar.gz: a44b90c9a612b18b1da300622d61e923c72afe566d881103b07a587238a03a6e695ec1b3af51850be39f8895baf67f836e7e83f38662228afe93723d70a714ee

data/bin/tailf2kafka

@@ -9,6 +9,8 @@ require 'rb-inotify'
 require 'timers'
 require 'socket'
 require 'fileutils'
+require 'logger'
+require 'mixlib/shellout'
 
 $stdout.sync = true
 
@@ -16,9 +18,21 @@ Thread.abort_on_exception = true
 
 @config = nil
 
+loglevels = {
+  :debug => Logger::DEBUG,
+  :info => Logger::INFO,
+  :warn => Logger::WARN,
+  :error => Logger::Error,
+  :fatal => Logger::FATAL,
+  :unknown => Logger::UNKNOWN
+}
+
+@loglevel = Logger::INFO
+
 opts = OptionParser.new
 opts.banner = "Usage: #{$0} [options]"
 opts.on( '--config PATH', String, 'Path to settings config' ) { |c| @config = c }
+opts.on( '--log-level [LEVEL]', [:debug, :info, :warn, :error, :fatal, :unknown] ) { |l| @loglevel = loglevels[l] }
 opts.on( '-h', '--help', 'Display this screen' ) { puts opts; exit 0 }
 opts.parse!
 
@@ -27,6 +41,8 @@ unless @config
   exit 1
 end
 
+@logger = Logger.new(STDOUT)
+
 @settings = YAML.load_file(@config).symbolize_keys(true)
 
 validator = Schash::Validator.new do
@@ -43,6 +59,7 @@ validator = Schash::Validator.new do
       max_batch_lines: integer,
       from_begining: boolean,
       delete_old_tailed_files: optional(boolean),
+      post_delete_command: optional(string),
     },
     kafka: {
       brokers: array_of(string),
@@ -53,8 +70,8 @@ validator = Schash::Validator.new do
 end
 
 unless validator.validate(@settings).empty?
-  puts "ERROR: bad settings "
-  pp validator.validate(@settings)
+  @logger.error("ERROR: bad settings")
+  @logger.error(validator.validate(@settings))
   exit 1
 end
 
@@ -114,7 +131,7 @@ load_position_file
     begin
       @producer.send_messages(batch[:messages]) if @produce
     rescue Poseidon::Errors::UnableToFetchMetadata
-      puts "Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ..."
+      @logger.warn("Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ...")
       sleep 1
       retry
     end
@@ -123,31 +140,52 @@ load_position_file
 end
 
 def kafka_produce(path, buffer, offset)
+  truncated = nil
+
   messages = []
-  buffer.each do |msg|
-    messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
+  while msg = buffer.shift
+    unless msg[-1] == "\n"
+      if buffer.empty?
+        truncated = msg
+      else
+        msg = msg + buffer.shift
+        messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
+      end
+    else
+      messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
+    end
   end
   @producer_queue.push({ :path => path, :messages => messages, :offset => offset})
+
+  truncated
 end
 
 def tailf(path)
   file = File.open(path, 'r')
   @files[path][:fd] = file
   file.seek(@files[path][:offset], IO::SEEK_SET)
+
+  truncated = nil
   loop do #Fast read file in batches until we reach EOF upon which we start the tailf modify watcher
     batch = file.each_line.take(@max_batch_lines)
     break if batch.empty?
-    kafka_produce(path, batch, file.pos)
+    batch.unshift(truncated + batch.shift) if truncated
+    truncated = kafka_produce(path, batch, file.pos)
   end
+
+  mutex = Mutex.new
   @tailf_notifier.watch(path, :modify) do |event|
-    unless file.closed?
-      loop do
-        batch = file.each_line.take(@max_batch_lines)
-        break if batch.empty?
-        kafka_produce(path, batch, file.pos)
+    mutex.synchronize do
+      unless file.closed?
+        loop do
+          batch = file.each_line.take(@max_batch_lines)
+          break if batch.empty?
+          batch.unshift(truncated + batch.shift) if truncated
+          truncated = kafka_produce(path, batch, file.pos)
+        end
+      else
+        @logger.warn("watcher got modify event on closed file #{event.name}")
       end
-    else
-      puts "watcher got modify event on closed file #{event.name}"
     end
   end
 end
@@ -192,8 +230,12 @@ def delete_old_tailed_files
     @files.each_key do |path|
       unless path.match(Regexp.new(Time.now.strftime(@files[path][:pattern])))
         if File.exists?(path) and File.stat(path).ino == @files[path][:inode] and File.stat(path).size == @files[path][:offset] and (Time.now - File.stat(path).mtime) > 30
-          puts "Deleteing old time pattern fully kafka produced file #{path}"
+          @logger.info("Deleteing old time pattern fully kafka produced file #{path}")
           FileUtils.rm_r(path)
+          if @settings[:tailf].has_key?(:post_delete_command)
+            command = Mixlib::ShellOut.new(@settings[:tailf][:post_delete_command])
+            command.run_command
+          end
         end
       end
     end
@@ -214,7 +256,7 @@ Thread.new { loop { @timers.wait } }
       if match
         unless File.directory?(path)
           unless @threads.has_key?(path)
-            puts "File #{event.name} was created in / moved into watched dir #{dir}"
+            @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
             @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
             @threads[path] = Thread.new { tailf(path) }
           end
@@ -227,7 +269,7 @@ Thread.new { loop { @timers.wait } }
     @mutex.synchronize do
       path = "#{dir}/#{event.name}"
       if @threads.has_key?(path)
-        puts "File #{event.name} was deleted / moved from watched dir #{dir}"
+        @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
         if @threads[path].alive?
           @threads[path].terminate
           @threads[path].join

data/lib/tailf2kafka/version.rb

@@ -1,3 +1,3 @@
 module Tailf2Kafka
-  VERSION ||= '0.1.4'
+  VERSION ||= '0.1.5'
 end

data/tailf2kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.summary     = "Watch and tail files with specified time based patterns and push them to kafka"
   s.description = "Watch and tail files with specified time based patterns and push them to kafka"
   s.license     = 'MIT'
-  s.has_rdoc    = false 
+  s.has_rdoc    = false
 
   s.add_dependency('poseidon')
   s.add_dependency('snappy')
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.add_dependency('schash')
   s.add_dependency('rb-inotify')
   s.add_dependency('timers')
+  s.add_dependency('mixlib-shellout')
 
   s.add_development_dependency('rake')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tailf2kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Alexander Piavlo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-19 00:00:00.000000000 Z
+date: 2015-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mixlib-shellout
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement