tailf2kafka 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a52bb6e6124032b75f8bd0ad29f6e3ea31e4d43
-  data.tar.gz: d0cf9ebc419b52d2a67c4de51e6ace18637c4212
+  metadata.gz: 36ed03a088f2e37c9e35773e233159bc1c32cf2d
+  data.tar.gz: 203b2fd8a437e4d8baa555f46a246411eed0cb9e
 SHA512:
-  metadata.gz: 4fc9d64f58bdb1bd22da9dc45c1f7a1c7c62cd846fc596542613b92e7baa1e1f1d95c718e4c79cce66cfd91d39fa27f8ee7ee8c3b28fe4ab5a0497951a6b1b9b
-  data.tar.gz: d4cfb41c0a41f0a905f4d68167a42aea8293d5f7a3757cf60a307d3e81a73ec85a49a8b6628a6ac9aecc76a410c3f7ae4fa9c0b2b5ca090be6938e19d2548315
+  metadata.gz: 5554c921a0ce29a3bf4f3548e8b180a46545ace278163b4cebaff11255e709565730f5de25263fc4fa44636a345eb6eec2a187e7dd012dc16bdd109bbb02c484
+  data.tar.gz: 11bc611eac30966753a8327acc0a7f97304d6b1c8757893ef7316b6cf8f1acfc83fdc7a9981b46ccb720a63bb3e5940657ddc8d4ed3795b838d590c8f3c1d485

data/bin/tailf2kafka

@@ -205,7 +205,7 @@ def time_pattern_to_regexp(pattern)
   end
 end
 
-#Scan existing files that match watched prefixes and start failing them
+#Scan existing files that match watched prefixes and start tailing them
 @settings[:tailf][:files].each do |tailf_file|
   tailf_file[:prefix] = File.expand_path(tailf_file[:prefix])
   dir = File.dirname(tailf_file[:prefix])
@@ -257,46 +257,52 @@ end
 
 @timers = Timers::Group.new
 @uploads_timer = @timers.every(@flush_interval) { write_position_file }
-@delete_old_tailed_files_timer = @timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
+@timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
 Thread.new { loop { @timers.wait } }
 
-@dirs.each_key do |dir|
-
-  @create_notifier.watch(dir, :create, :moved_to) do |event|
-    @mutex.synchronize do
-      path = "#{dir}/#{event.name}"
-      match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
-      if match
-        unless File.directory?(path)
-          unless @threads.has_key?(path)
-            @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
-            @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
-            @threads[path] = Thread.new { tailf(path) }
+def setup_watchers
+  @dirs.each_key do |dir|
+
+    @create_notifier.watch(dir, :create, :moved_to) do |event|
+      @mutex.synchronize do
+        path = "#{dir}/#{event.name}"
+        match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
+        if match
+          unless File.directory?(path)
+            unless @threads.has_key?(path)
+              @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
+              @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
+              @threads[path] = Thread.new { tailf(path) }
+            end
           end
         end
       end
     end
-  end
 
-  @delete_notifier.watch(dir, :delete, :moved_from) do |event|
-    @mutex.synchronize do
-      path = "#{dir}/#{event.name}"
-      if @threads.has_key?(path)
-        @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
-        if @threads[path].alive?
-          @threads[path].terminate
-          @threads[path].join
+    @delete_notifier.watch(dir, :delete, :moved_from) do |event|
+      @mutex.synchronize do
+        path = "#{dir}/#{event.name}"
+        if @threads.has_key?(path)
+          @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
+          if @threads[path].alive?
+            @threads[path].terminate
+            @threads[path].join
+          end
+          @threads.delete(path)
+          @files[path][:fd].close unless @files[path][:fd].closed?
+          @files.delete(path)
         end
-        @threads.delete(path)
-        @files[path][:fd].close unless @files[path][:fd].closed?
-        @files.delete(path)
       end
     end
-  end
 
+  end
 end
 
+setup_watchers
+
 Thread.new { @create_notifier.run }
 Thread.new { @delete_notifier.run }
 
+@timers.every(60) { setup_watchers }
+
 @tailf_notifier.run

data/bin/tailf2kafka.save

@@ -0,0 +1,302 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'poseidon'
+require 'yaml'
+require 'hash_symbolizer'
+require 'schash'
+require 'rb-inotify'
+require 'timers'
+require 'socket'
+require 'fileutils'
+require 'logger'
+require 'mixlib/shellout'
+
+$stdout.sync = true
+
+Thread.abort_on_exception = true
+
+@config = nil
+
+loglevels = {
+  :debug => Logger::DEBUG,
+  :info => Logger::INFO,
+  :warn => Logger::WARN,
+  :error => Logger::Error,
+  :fatal => Logger::FATAL,
+  :unknown => Logger::UNKNOWN
+}
+
+@loglevel = Logger::INFO
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on( '--config PATH', String, 'Path to settings config' ) { |c| @config = c }
+opts.on( '--log-level [LEVEL]', [:debug, :info, :warn, :error, :fatal, :unknown] ) { |l| @loglevel = loglevels[l] }
+opts.on( '-h', '--help', 'Display this screen' ) { puts opts; exit 0 }
+opts.parse!
+
+unless @config
+  puts opts
+  exit 1
+end
+
+@logger = Logger.new(STDOUT)
+
+@settings = YAML.load_file(@config).symbolize_keys(true)
+
+validator = Schash::Validator.new do
+  {
+    tailf: {
+      files: array_of({
+        topic: string,
+        prefix: string,
+        suffix: optional(string),
+        time_pattern: string,
+      }),
+      position_file: string,
+      flush_interval: integer,
+      max_batch_lines: integer,
+      from_begining: boolean,
+      delete_old_tailed_files: optional(boolean),
+      post_delete_command: optional(string),
+    },
+    kafka: {
+      brokers: array_of(string),
+      producer_type: match(/^(sync|async)$/),
+      produce: optional(boolean),
+    },
+  }
+end
+
+unless validator.validate(@settings).empty?
+  @logger.error("ERROR: bad settings")
+  @logger.error(validator.validate(@settings))
+  exit 1
+end
+
+@settings[:tailf][:files] = @settings[:tailf][:files].map{|h| h.symbolize_keys(true)}
+
+@mutex = Mutex.new
+
+@create_notifier = INotify::Notifier.new
+@delete_notifier = INotify::Notifier.new
+@tailf_notifier = INotify::Notifier.new
+
+@dirs = {}
+@files = {}
+@threads = {}
+@position_file = @settings[:tailf][:position_file]
+@flush_interval = @settings[:tailf][:flush_interval]
+@max_batch_lines = @settings[:tailf][:max_batch_lines]
+@from_begining = @settings[:tailf][:from_begining]
+@delete_old_tailed_files = @settings[:tailf].has_key?(:delete_old_tailed_files) ?  @settings[:tailf][:delete_old_tailed_files] : false
+@brokers = @settings[:kafka][:brokers]
+@producer_type = @settings[:kafka][:producer_type].to_sym
+@produce = @settings[:kafka].has_key?(:produce) ? @settings[:kafka][:produce] : true
+
+def write_position_file
+  @mutex.synchronize do
+    File.open(@position_file, 'w') do |file|
+      @files.each do |path, attrs|
+        file.puts "#{path} #{attrs[:pattern]} #{attrs[:topic]} #{attrs[:inode]} #{attrs[:offset]}"
+      end
+    end
+  end
+end
+
+def load_position_file
+  if File.exist?(@position_file)
+    IO.readlines(@position_file).each do |line|
+      path, pattern, topic, inode, offset = line.split(' ')
+      #Load state only for that exist with same inode and were not truncated/rewinded.
+      if File.exists?(path) and File.stat(path).ino == inode.to_i and File.stat(path).size >= offset.to_i
+        @files[path] = { :pattern => pattern, :topic => topic, :inode => inode.to_i, :offset => offset.to_i }
+      end
+    end
+  end
+  write_position_file
+end
+
+load_position_file
+
+@topics = @settings[:tailf][:files].map{|tailf_file| tailf_file[:topic]}
+@producer = Poseidon::Producer.new(@brokers, "#{Socket.gethostname}", :type => @producer_type, :compression_codec => :snappy, :compressed_topics => @topics) if @produce
+
+@producer_queue = SizedQueue.new(10)
+
+@producer_thread = Thread.new do
+  loop do
+    batch = @producer_queue.pop
+    begin
+      @producer.send_messages(batch[:messages]) if @produce
+    rescue Poseidon::Errors::UnableToFetchMetadata
+      @logger.warn("Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ...")
+      sleep 1
+      retry
+    end
+    @files[batch[:path]][:offset] = batch[:offset]
+  end
+end
+
+def kafka_produce(path, buffer, offset)
+  truncated = nil
+
+  messages = []
+  while msg = buffer.shift
+    unless msg[-1] == "\n"
+      if buffer.empty?
+        truncated = msg
+      else
+        msg = msg + buffer.shift
+        messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
+      end
+    else
+      messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
+    end
+  end
+  @producer_queue.push({ :path => path, :messages => messages, :offset => offset})
+
+  truncated
+end
+
+def tailf(path)
+  file = File.open(path, 'r')
+  @files[path][:fd] = file
+  file.seek(@files[path][:offset], IO::SEEK_SET)
+
+  truncated = nil
+  loop do #Fast read file in batches until we reach EOF upon which we start the tailf modify watcher
+    batch = file.each_line.take(@max_batch_lines)
+    break if batch.empty?
+    batch.unshift(truncated + batch.shift) if truncated
+    truncated = kafka_produce(path, batch, file.pos)
+  end
+
+  mutex = Mutex.new
+  @tailf_notifier.watch(path, :modify) do |event|
+    mutex.synchronize do
+      unless file.closed?
+        loop do
+          batch = file.each_line.take(@max_batch_lines)
+          break if batch.empty?
+          batch.unshift(truncated + batch.shift) if truncated
+          truncated = kafka_produce(path, batch, file.pos)
+        end
+      else
+        @logger.warn("watcher got modify event on closed file #{event.name}")
+      end
+    end
+  end
+end
+
+@time_regexp_hash = {
+  'Y' => '[0-9]{4}',
+  'm' => '[0-9]{2}',
+  'd' => '[0-9]{2}',
+  'H' => '[0-9]{2}',
+  'M' => '[0-9]{2}'
+}
+
+def time_pattern_to_regexp(pattern)
+  pattern.gsub(/%([^%])/) do
+    match = $1
+    @time_regexp_hash.has_key?(match) ? @time_regexp_hash[match] : match
+  end
+end
+
+#Scan existing files that match watched prefixes and start failing them
+@settings[:tailf][:files].each do |tailf_file|
+  dir = File.dirname(tailf_file[:prefix])
+  if File.exists?(dir) and File.directory?(dir)
+    @dirs[dir] ||= []
+    @dirs[dir] << { :prefix => File.basename(tailf_file[:prefix]), :pattern => tailf_file[:time_pattern], :suffix => "#{tailf_file[:suffix]}", :topic => tailf_file[:topic]}
+    Dir.glob("#{tailf_file[:prefix]}*#{tailf_file[:suffix]}").each do |path|
+      if path.match(Regexp.new(time_pattern_to_regexp(tailf_file[:time_pattern])))
+        unless File.directory?(path)
+          #Populate state only if it was not loaded from position file
+          unless @files.has_key?(path)
+            @files[path] = { :pattern => tailf_file[:time_pattern], :topic => tailf_file[:topic], :inode => File.stat(path).ino, :offset => 0 }
+            @files[path][:offset] = File.stat(path).size unless @from_begining
+          end
+          @threads[path] = Thread.new { tailf(path) } unless @threads.has_key?(path)
+        end
+      end
+    end
+  end
+end
+
+def delete_old_tailed_files
+  @mutex.synchronize do
+    @files.each_key do |path|
+      unless path.match(Regexp.new(Time.now.strftime(@files[path][:pattern])))
+        if File.exists?(path) and File.stat(path).ino == @files[path][:inode] and File.stat(path).size == @files[path][:offset] and (Time.now - File.stat(path).mtime) > 30
+          @logger.info("Deleteing old time pattern fully kafka produced file #{path}")
+          FileUtils.rm_r(path)
+          if @settings[:tailf].has_key?(:post_delete_command)
+            @logger.info("Running post delete command => #{@settings[:tailf][:post_delete_command]}")
+            command = Mixlib::created
+ShellOut.new(@settings[:tailf][:post_delete_command])
+            begin
+              command.run_command
+              if command.error?
+                @logger.error("Failed post delete command => #{@settings[:tailf][:post_delete_command]}")
+                @logger.info("STDOUT: #{command.stdout}")
+                @logger.info("STDERR: #{command.stderr}")
+              end
+            rescue => e
+              @logger.error("Failed post delete command => #{@settings[:tailf][:post_delete_command]}")
+              @logger.info(e.message)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+@timers = Timers::Group.new
+@uploads_timer = @timers.every(@flush_interval) { write_position_file }
+@delete_old_tailed_files_timer = @timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
+Thread.new { loop { @timers.wait } }
+
+@dirs.each_key do |dir|
+
+  @create_notifier.watch(dir, :create, :moved_to) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
+      if match
+        unless File.directory?(path)
+          unless @threads.has_key?(path)
+            @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
+            @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
+            @threads[path] = Thread.new { tailf(path) }
+          end
+        end
+      end
+    end
+  end
+
+  @delete_notifier.watch(dir, :delete, :moved_from) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      if @threads.has_key?(path)
+        @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
+        if @threads[path].alive?
+          @threads[path].terminate
+          @threads[path].join
+        end
+        @threads.delete(path)
+        @files[path][:fd].close unless @files[path][:fd].closed?
+        @files.delete(path)
+      end
+    end
+  end
+
+end
+
+Thread.new { @create_notifier.run }
+Thread.new { @delete_notifier.run }
+
+@tailf_notifier.run

data/lib/tailf2kafka/version.rb

@@ -1,3 +1,3 @@
 module Tailf2Kafka
-  VERSION ||= '0.1.7'
+  VERSION ||= '0.1.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tailf2kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Alexander Piavlo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-29 00:00:00.000000000 Z
+date: 2016-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon
@@ -128,12 +128,14 @@ email:
 - devops@supersonic.com
 executables:
 - tailf2kafka
+- tailf2kafka.save
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
 - bin/tailf2kafka
+- bin/tailf2kafka.save
 - lib/tailf2kafka.rb
 - lib/tailf2kafka/version.rb
 - tailf2kafka.gemspec