tailf2kafka 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36ed03a088f2e37c9e35773e233159bc1c32cf2d
-  data.tar.gz: 203b2fd8a437e4d8baa555f46a246411eed0cb9e
+  metadata.gz: 8eef4e08a12fcd549df3c091eb06a5784669dfa0
+  data.tar.gz: 953bd871eba9fdfa942764464238fa5560c16d2e
 SHA512:
-  metadata.gz: 5554c921a0ce29a3bf4f3548e8b180a46545ace278163b4cebaff11255e709565730f5de25263fc4fa44636a345eb6eec2a187e7dd012dc16bdd109bbb02c484
-  data.tar.gz: 11bc611eac30966753a8327acc0a7f97304d6b1c8757893ef7316b6cf8f1acfc83fdc7a9981b46ccb720a63bb3e5940657ddc8d4ed3795b838d590c8f3c1d485
+  metadata.gz: a79f4f802ea67c650c673513dcdfb8f033f448264a02bc1ce75a5eab335b86db35313ac15ad6402a152eea0427ce000586453fb8aac1eeaf5ead625cca5349be
+  data.tar.gz: 0f353df4ab74ad941e6857ef8611209c4b976394ce811b538bd8e7eb58dfda0187bd892bb78728edfe61e79f76d9de5f86336caa156f3eba330681f3c607fb7e

data/bin/tailf2kafka

@@ -205,7 +205,7 @@ def time_pattern_to_regexp(pattern)
   end
 end
 
-#Scan existing files that match watched prefixes and start tailing them
+#Scan existing files that match watched prefixes and start failing them
 @settings[:tailf][:files].each do |tailf_file|
   tailf_file[:prefix] = File.expand_path(tailf_file[:prefix])
   dir = File.dirname(tailf_file[:prefix])
@@ -257,52 +257,46 @@ end
 
 @timers = Timers::Group.new
 @uploads_timer = @timers.every(@flush_interval) { write_position_file }
-@timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
+@delete_old_tailed_files_timer = @timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
 Thread.new { loop { @timers.wait } }
 
-def setup_watchers
-  @dirs.each_key do |dir|
-
-    @create_notifier.watch(dir, :create, :moved_to) do |event|
-      @mutex.synchronize do
-        path = "#{dir}/#{event.name}"
-        match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
-        if match
-          unless File.directory?(path)
-            unless @threads.has_key?(path)
-              @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
-              @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
-              @threads[path] = Thread.new { tailf(path) }
-            end
+@dirs.each_key do |dir|
+
+  @create_notifier.watch(dir, :create, :moved_to) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
+      if match
+        unless File.directory?(path)
+          unless @threads.has_key?(path)
+            @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
+            @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
+            @threads[path] = Thread.new { tailf(path) }
           end
         end
       end
     end
+  end
 
-    @delete_notifier.watch(dir, :delete, :moved_from) do |event|
-      @mutex.synchronize do
-        path = "#{dir}/#{event.name}"
-        if @threads.has_key?(path)
-          @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
-          if @threads[path].alive?
-            @threads[path].terminate
-            @threads[path].join
-          end
-          @threads.delete(path)
-          @files[path][:fd].close unless @files[path][:fd].closed?
-          @files.delete(path)
+  @delete_notifier.watch(dir, :delete, :moved_from) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      if @threads.has_key?(path)
+        @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
+        if @threads[path].alive?
+          @threads[path].terminate
+          @threads[path].join
         end
+        @threads.delete(path)
+        @files[path][:fd].close unless @files[path][:fd].closed?
+        @files.delete(path)
       end
     end
-
   end
-end
 
-setup_watchers
+end
 
 Thread.new { @create_notifier.run }
 Thread.new { @delete_notifier.run }
 
-@timers.every(60) { setup_watchers }
-
 @tailf_notifier.run

data/lib/tailf2kafka/version.rb

@@ -1,3 +1,3 @@
 module Tailf2Kafka
-  VERSION ||= '0.1.8'
+  VERSION ||= '0.1.9'
 end

data/tailf2kafka.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   s.add_dependency('rb-inotify')
   s.add_dependency('timers')
   s.add_dependency('mixlib-shellout')
+  s.add_dependency('activesupport', '~> 4.2.6')
 
   s.add_development_dependency('rake')
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tailf2kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Alexander Piavlo
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.6
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +142,12 @@ email:
 - devops@supersonic.com
 executables:
 - tailf2kafka
-- tailf2kafka.save
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
 - bin/tailf2kafka
-- bin/tailf2kafka.save
 - lib/tailf2kafka.rb
 - lib/tailf2kafka/version.rb
 - tailf2kafka.gemspec
@@ -159,9 +171,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Watch and tail files with specified time based patterns and push them to
   kafka
 test_files: []
+has_rdoc: false

data/bin/tailf2kafka.save

@@ -1,302 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'optparse'
-require 'poseidon'
-require 'yaml'
-require 'hash_symbolizer'
-require 'schash'
-require 'rb-inotify'
-require 'timers'
-require 'socket'
-require 'fileutils'
-require 'logger'
-require 'mixlib/shellout'
-
-$stdout.sync = true
-
-Thread.abort_on_exception = true
-
-@config = nil
-
-loglevels = {
-  :debug => Logger::DEBUG,
-  :info => Logger::INFO,
-  :warn => Logger::WARN,
-  :error => Logger::Error,
-  :fatal => Logger::FATAL,
-  :unknown => Logger::UNKNOWN
-}
-
-@loglevel = Logger::INFO
-
-opts = OptionParser.new
-opts.banner = "Usage: #{$0} [options]"
-opts.on( '--config PATH', String, 'Path to settings config' ) { |c| @config = c }
-opts.on( '--log-level [LEVEL]', [:debug, :info, :warn, :error, :fatal, :unknown] ) { |l| @loglevel = loglevels[l] }
-opts.on( '-h', '--help', 'Display this screen' ) { puts opts; exit 0 }
-opts.parse!
-
-unless @config
-  puts opts
-  exit 1
-end
-
-@logger = Logger.new(STDOUT)
-
-@settings = YAML.load_file(@config).symbolize_keys(true)
-
-validator = Schash::Validator.new do
-  {
-    tailf: {
-      files: array_of({
-        topic: string,
-        prefix: string,
-        suffix: optional(string),
-        time_pattern: string,
-      }),
-      position_file: string,
-      flush_interval: integer,
-      max_batch_lines: integer,
-      from_begining: boolean,
-      delete_old_tailed_files: optional(boolean),
-      post_delete_command: optional(string),
-    },
-    kafka: {
-      brokers: array_of(string),
-      producer_type: match(/^(sync|async)$/),
-      produce: optional(boolean),
-    },
-  }
-end
-
-unless validator.validate(@settings).empty?
-  @logger.error("ERROR: bad settings")
-  @logger.error(validator.validate(@settings))
-  exit 1
-end
-
-@settings[:tailf][:files] = @settings[:tailf][:files].map{|h| h.symbolize_keys(true)}
-
-@mutex = Mutex.new
-
-@create_notifier = INotify::Notifier.new
-@delete_notifier = INotify::Notifier.new
-@tailf_notifier = INotify::Notifier.new
-
-@dirs = {}
-@files = {}
-@threads = {}
-@position_file = @settings[:tailf][:position_file]
-@flush_interval = @settings[:tailf][:flush_interval]
-@max_batch_lines = @settings[:tailf][:max_batch_lines]
-@from_begining = @settings[:tailf][:from_begining]
-@delete_old_tailed_files = @settings[:tailf].has_key?(:delete_old_tailed_files) ?  @settings[:tailf][:delete_old_tailed_files] : false
-@brokers = @settings[:kafka][:brokers]
-@producer_type = @settings[:kafka][:producer_type].to_sym
-@produce = @settings[:kafka].has_key?(:produce) ? @settings[:kafka][:produce] : true
-
-def write_position_file
-  @mutex.synchronize do
-    File.open(@position_file, 'w') do |file|
-      @files.each do |path, attrs|
-        file.puts "#{path} #{attrs[:pattern]} #{attrs[:topic]} #{attrs[:inode]} #{attrs[:offset]}"
-      end
-    end
-  end
-end
-
-def load_position_file
-  if File.exist?(@position_file)
-    IO.readlines(@position_file).each do |line|
-      path, pattern, topic, inode, offset = line.split(' ')
-      #Load state only for that exist with same inode and were not truncated/rewinded.
-      if File.exists?(path) and File.stat(path).ino == inode.to_i and File.stat(path).size >= offset.to_i
-        @files[path] = { :pattern => pattern, :topic => topic, :inode => inode.to_i, :offset => offset.to_i }
-      end
-    end
-  end
-  write_position_file
-end
-
-load_position_file
-
-@topics = @settings[:tailf][:files].map{|tailf_file| tailf_file[:topic]}
-@producer = Poseidon::Producer.new(@brokers, "#{Socket.gethostname}", :type => @producer_type, :compression_codec => :snappy, :compressed_topics => @topics) if @produce
-
-@producer_queue = SizedQueue.new(10)
-
-@producer_thread = Thread.new do
-  loop do
-    batch = @producer_queue.pop
-    begin
-      @producer.send_messages(batch[:messages]) if @produce
-    rescue Poseidon::Errors::UnableToFetchMetadata
-      @logger.warn("Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ...")
-      sleep 1
-      retry
-    end
-    @files[batch[:path]][:offset] = batch[:offset]
-  end
-end
-
-def kafka_produce(path, buffer, offset)
-  truncated = nil
-
-  messages = []
-  while msg = buffer.shift
-    unless msg[-1] == "\n"
-      if buffer.empty?
-        truncated = msg
-      else
-        msg = msg + buffer.shift
-        messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
-      end
-    else
-      messages << Poseidon::MessageToSend.new(@files[path][:topic], msg.strip)
-    end
-  end
-  @producer_queue.push({ :path => path, :messages => messages, :offset => offset})
-
-  truncated
-end
-
-def tailf(path)
-  file = File.open(path, 'r')
-  @files[path][:fd] = file
-  file.seek(@files[path][:offset], IO::SEEK_SET)
-
-  truncated = nil
-  loop do #Fast read file in batches until we reach EOF upon which we start the tailf modify watcher
-    batch = file.each_line.take(@max_batch_lines)
-    break if batch.empty?
-    batch.unshift(truncated + batch.shift) if truncated
-    truncated = kafka_produce(path, batch, file.pos)
-  end
-
-  mutex = Mutex.new
-  @tailf_notifier.watch(path, :modify) do |event|
-    mutex.synchronize do
-      unless file.closed?
-        loop do
-          batch = file.each_line.take(@max_batch_lines)
-          break if batch.empty?
-          batch.unshift(truncated + batch.shift) if truncated
-          truncated = kafka_produce(path, batch, file.pos)
-        end
-      else
-        @logger.warn("watcher got modify event on closed file #{event.name}")
-      end
-    end
-  end
-end
-
-@time_regexp_hash = {
-  'Y' => '[0-9]{4}',
-  'm' => '[0-9]{2}',
-  'd' => '[0-9]{2}',
-  'H' => '[0-9]{2}',
-  'M' => '[0-9]{2}'
-}
-
-def time_pattern_to_regexp(pattern)
-  pattern.gsub(/%([^%])/) do
-    match = $1
-    @time_regexp_hash.has_key?(match) ? @time_regexp_hash[match] : match
-  end
-end
-
-#Scan existing files that match watched prefixes and start failing them
-@settings[:tailf][:files].each do |tailf_file|
-  dir = File.dirname(tailf_file[:prefix])
-  if File.exists?(dir) and File.directory?(dir)
-    @dirs[dir] ||= []
-    @dirs[dir] << { :prefix => File.basename(tailf_file[:prefix]), :pattern => tailf_file[:time_pattern], :suffix => "#{tailf_file[:suffix]}", :topic => tailf_file[:topic]}
-    Dir.glob("#{tailf_file[:prefix]}*#{tailf_file[:suffix]}").each do |path|
-      if path.match(Regexp.new(time_pattern_to_regexp(tailf_file[:time_pattern])))
-        unless File.directory?(path)
-          #Populate state only if it was not loaded from position file
-          unless @files.has_key?(path)
-            @files[path] = { :pattern => tailf_file[:time_pattern], :topic => tailf_file[:topic], :inode => File.stat(path).ino, :offset => 0 }
-            @files[path][:offset] = File.stat(path).size unless @from_begining
-          end
-          @threads[path] = Thread.new { tailf(path) } unless @threads.has_key?(path)
-        end
-      end
-    end
-  end
-end
-
-def delete_old_tailed_files
-  @mutex.synchronize do
-    @files.each_key do |path|
-      unless path.match(Regexp.new(Time.now.strftime(@files[path][:pattern])))
-        if File.exists?(path) and File.stat(path).ino == @files[path][:inode] and File.stat(path).size == @files[path][:offset] and (Time.now - File.stat(path).mtime) > 30
-          @logger.info("Deleteing old time pattern fully kafka produced file #{path}")
-          FileUtils.rm_r(path)
-          if @settings[:tailf].has_key?(:post_delete_command)
-            @logger.info("Running post delete command => #{@settings[:tailf][:post_delete_command]}")
-            command = Mixlib::created
-ShellOut.new(@settings[:tailf][:post_delete_command])
-            begin
-              command.run_command
-              if command.error?
-                @logger.error("Failed post delete command => #{@settings[:tailf][:post_delete_command]}")
-                @logger.info("STDOUT: #{command.stdout}")
-                @logger.info("STDERR: #{command.stderr}")
-              end
-            rescue => e
-              @logger.error("Failed post delete command => #{@settings[:tailf][:post_delete_command]}")
-              @logger.info(e.message)
-            end
-          end
-        end
-      end
-    end
-  end
-end
-
-@timers = Timers::Group.new
-@uploads_timer = @timers.every(@flush_interval) { write_position_file }
-@delete_old_tailed_files_timer = @timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
-Thread.new { loop { @timers.wait } }
-
-@dirs.each_key do |dir|
-
-  @create_notifier.watch(dir, :create, :moved_to) do |event|
-    @mutex.synchronize do
-      path = "#{dir}/#{event.name}"
-      match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
-      if match
-        unless File.directory?(path)
-          unless @threads.has_key?(path)
-            @logger.info("File #{event.name} was created in / moved into watched dir #{dir}")
-            @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
-            @threads[path] = Thread.new { tailf(path) }
-          end
-        end
-      end
-    end
-  end
-
-  @delete_notifier.watch(dir, :delete, :moved_from) do |event|
-    @mutex.synchronize do
-      path = "#{dir}/#{event.name}"
-      if @threads.has_key?(path)
-        @logger.info("File #{event.name} was deleted / moved from watched dir #{dir}")
-        if @threads[path].alive?
-          @threads[path].terminate
-          @threads[path].join
-        end
-        @threads.delete(path)
-        @files[path][:fd].close unless @files[path][:fd].closed?
-        @files.delete(path)
-      end
-    end
-  end
-
-end
-
-Thread.new { @create_notifier.run }
-Thread.new { @delete_notifier.run }
-
-@tailf_notifier.run