tailf2kafka 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0511b332f01cad011509e5312d35a44406996394
+  data.tar.gz: 1ca828eed00c80a7896b3cf384292e577d8832f5
+SHA512:
+  metadata.gz: bbe8203735d60e6483a5aef2f0a3f6c0473828753fa5d4649313b56a147b96e63fa300bec9f6a5900995c6c56578b69e4db6af88d572c1e946fbb8a687508f56
+  data.tar.gz: 1e9c1d14ad055d267a005208a5a2a59bd9a5786984bda8b323aff08b6703a75c596761ce688c9ec1552d8fd3a15dc6e5b78e1090ab7e9dd740890b937ab85997

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Supersonic LTD
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# Tailf2Kafka
+
+Watch and tail files in dirs with specified filename time based patterns and push them to kafka.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'tailf2kafka'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install tailf2kafka
+
+## Usage
+
+    $ tailf2kafka -h
+    Usage: tailf2kafka [options]
+            --config PATH                Path to settings config
+        -h, --help                       Display this screen
+    $
+
+## Config
+
+    tailf:
+      files:
+        - topic: haproxy
+          prefix: /var/log/haproxy/haproxy
+          time_pattern: ".%Y-%m-%d.%H"
+      position_file: "/var/lib/haproxy/tail2kafka.offsets"
+      flush_interval: 1
+      max_batch_lines: 1024
+      from_begining: true
+      delete_old_tailed_files: true
+    kafka:
+      brokers: ["broker1:9092", "broker2:9092", "broker3:9092"]
+      producer_type: sync
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+6. Go to 1

data/bin/tailf2kafka

@@ -0,0 +1,244 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'poseidon'
+require 'yaml'
+require 'hash_symbolizer'
+require 'schash'
+require 'rb-inotify'
+require 'timers'
+require 'socket'
+require 'fileutils'
+
+$stdout.sync = true
+
+Thread.abort_on_exception = true
+
+@config = nil
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on( '--config PATH', String, 'Path to settings config' ) { |c| @config = c }
+opts.on( '-h', '--help', 'Display this screen' ) { puts opts; exit 0 }
+opts.parse!
+
+unless @config
+  puts opts
+  exit 1
+end
+
+@settings = YAML.load_file(@config).symbolize_keys(true)
+
+validator = Schash::Validator.new do
+  {
+    tailf: {
+      files: array_of({
+        topic: string,
+        prefix: string,
+        suffix: optional(string),
+        time_pattern: string,
+      }),
+      position_file: string,
+      flush_interval: integer,
+      max_batch_lines: integer,
+      from_begining: boolean,
+      delete_old_tailed_files: optional(boolean),
+    },
+    kafka: {
+      brokers: array_of(string),
+      producer_type: match(/^(sync|async)$/),
+      produce: optional(boolean),
+    },
+  }
+end
+
+unless validator.validate(@settings).empty?
+  puts "ERROR: bad settings "
+  pp validator.validate(@settings)
+  exit 1
+end
+
+@settings[:tailf][:files] = @settings[:tailf][:files].map{|h| h.symbolize_keys(true)}
+
+@mutex = Mutex.new
+
+@create_notifier = INotify::Notifier.new
+@delete_notifier = INotify::Notifier.new
+@tailf_notifier = INotify::Notifier.new
+
+@dirs = {}
+@files = {}
+@threads = {}
+@position_file = @settings[:tailf][:position_file]
+@flush_interval = @settings[:tailf][:flush_interval]
+@max_batch_lines = @settings[:tailf][:max_batch_lines]
+@from_begining = @settings[:tailf][:from_begining]
+@delete_old_tailed_files = @settings[:tailf].has_key?(:delete_old_tailed_files) ?  @settings[:tailf][:delete_old_tailed_files] : false
+@brokers = @settings[:kafka][:brokers]
+@producer_type = @settings[:kafka][:producer_type].to_sym
+@produce = @settings[:kafka].has_key?(:produce) ? @settings[:kafka][:produce] : true
+
+def write_position_file
+  @mutex.synchronize do
+    File.open(@position_file, 'w') do |file|
+      @files.each do |path, attrs|
+        file.puts "#{path} #{attrs[:pattern]} #{attrs[:topic]} #{attrs[:inode]} #{attrs[:offset]}"
+      end
+    end
+  end
+end
+
+def load_position_file
+  if File.exist?(@position_file)
+    IO.readlines(@position_file).each do |line|
+      path, pattern, topic, inode, offset = line.split(' ')
+      #Load state only for that exist with same inode and were not truncated/rewinded.
+      if File.exists?(path) and File.stat(path).ino == inode.to_i and File.stat(path).size >= offset.to_i
+        @files[path] = { :pattern => pattern, :topic => topic, :inode => inode.to_i, :offset => offset.to_i }
+      end
+    end
+  end
+  write_position_file
+end
+
+load_position_file
+
+@topics = @settings[:tailf][:files].map{|tailf_file| tailf_file[:topic]}
+@producer = Poseidon::Producer.new(@brokers, "#{Socket.gethostname}", :type => @producer_type, :compression_codec => :snappy, :compressed_topics => @topics) if @produce
+
+@producer_queue = SizedQueue.new(10)
+
+@producer_thread = Thread.new do
+  loop do
+    batch = @producer_queue.pop
+    begin
+      @producer.send_messages(batch[:messages]) if @produce
+    rescue Poseidon::Errors::UnableToFetchMetadata
+      puts "Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ..."
+      sleep 1
+      retry
+    end
+    @files[batch[:path]][:offset] = batch[:offset]
+  end
+end
+
+def kafka_produce(path, buffer, offset)
+  messages = []
+  buffer.each do |msg|
+    messages << Poseidon::MessageToSend.new(@files[path][:topic], msg)
+  end
+  @producer_queue.push({ :path => path, :messages => messages, :offset => offset})
+end
+
+def tailf(path)
+  file = File.open(path, 'r')
+  @files[path][:fd] = file
+  file.seek(@files[path][:offset], IO::SEEK_SET)
+  loop do #Fast read file in batches until we reach EOF upon which we start the tailf modify watcher
+    batch = file.each_line.take(@max_batch_lines)
+    break if batch.empty?
+    kafka_produce(path, batch, file.pos)
+  end
+  @tailf_notifier.watch(path, :modify) do |event|
+    unless file.closed?
+      batch = file.each_line.take(@max_batch_lines)
+      kafka_produce(path, batch, file.pos) unless batch.empty?
+    else
+      puts "watcher got modify event on closed file #{event.name}"
+    end
+  end
+end
+
+@time_regexp_hash = {
+  'Y' => '[0-9]{4}',
+  'm' => '[0-9]{2}',
+  'd' => '[0-9]{2}',
+  'H' => '[0-9]{2}'
+}
+
+def time_pattern_to_regexp(pattern)
+  pattern.gsub(/%([^%])/) do
+    match = $1
+    @time_regexp_hash.has_key?(match) ? @time_regexp_hash[match] : match
+  end
+end
+
+#Scan existing files that match watched prefixes and start failing them
+@settings[:tailf][:files].each do |tailf_file|
+  dir = File.dirname(tailf_file[:prefix])
+  if File.exists?(dir) and File.directory?(dir)
+    @dirs[dir] ||= []
+    @dirs[dir] << { :prefix => File.basename(tailf_file[:prefix]), :pattern => tailf_file[:time_pattern], :suffix => "#{tailf_file[:suffix]}" }
+    Dir.glob("#{tailf_file[:prefix]}*#{tailf_file[:suffix]}").each do |path|
+      if path.match(Regexp.new(time_pattern_to_regexp(tailf_file[:time_pattern])))
+        unless File.directory?(path)
+          #Populate state only if it was not loaded from position file
+          unless @files.has_key?(path)
+            @files[path] = { :pattern => tailf_file[:time_pattern], :topic => tailf_file[:topic], :inode => File.stat(path).ino, :offset => 0 }
+            @files[path][:offset] = File.stat(path).size unless @from_begining
+          end
+          @threads[path] = Thread.new { tailf(path) } unless @threads.has_key?(path)
+        end
+      end
+    end
+  end
+end
+
+def delete_old_tailed_files
+  @mutex.synchronize do
+    @files.each_key do |path|
+      unless path.match(Regexp.new(Time.now.strftime(@files[path][:pattern])))
+        if File.exists?(path) and File.stat(path).ino == @files[path][:inode] and File.stat(path).size == @files[path][:offset]
+          puts "Deleteing old time pattern fully kafka produced file #{path}"
+          FileUtils.rm_r(path)
+        end
+      end
+    end
+  end
+end
+
+@timers = Timers::Group.new
+@uploads_timer = @timers.every(@flush_interval) { write_position_file }
+@delete_old_tailed_files_timer = @timers.every(60) { delete_old_tailed_files } if @delete_old_tailed_files
+Thread.new { loop { @timers.wait } }
+
+@dirs.each_key do |dir|
+
+  @create_notifier.watch(dir, :create, :moved_to) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      matches = @dirs[dir].select{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}.empty?
+      unless matches.empty?
+        unless File.directory?(path)
+          unless @threads.has_key?(path)
+            puts "File #{event.name} was created in / moved into watched dir #{dir}"
+            @files[path] = { :pattern => match.first[:pattern], :topic => File.basename(@dirs[dir].detect{|prefix| path.start_with?(prefix)}), :inode => File.stat(path).ino, :offset => 0 }
+            @threads[path] = Thread.new { tailf(path) }
+          end
+        end
+      end
+    end
+  end
+
+  @delete_notifier.watch(dir, :delete, :moved_from) do |event|
+    @mutex.synchronize do
+      path = "#{dir}/#{event.name}"
+      if @threads.has_key?(path)
+        puts "File #{event.name} was deleted / moved from watched dir #{dir}"
+        if @threads[path].alive?
+          @threads[path].terminate
+          @threads[path].join
+        end
+        @threads.delete(path)
+        @files[path][:fd].close unless @files[path][:fd].closed?
+        @files.delete(path)
+      end
+    end
+  end
+
+end
+
+Thread.new { @create_notifier.run }
+Thread.new { @delete_notifier.run }
+
+@tailf_notifier.run

data/lib/tailf2kafka/version.rb

@@ -0,0 +1,3 @@
+module Tailf2Kafka
+  VERSION ||= '0.1.0'
+end

data/lib/tailf2kafka.rb

@@ -0,0 +1,3 @@
+module Tailf2Kafka
+  require 'tailf2kafak/version.rb'
+end

data/tailf2kafka.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require "tailf2kafka/version"
+ 
+Gem::Specification.new do |s|
+  s.name        = "tailf2kafka"
+  s.version     = Tailf2Kafka::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Alexander Piavlo"]
+  s.email       = ["devops@supersonic.com"]
+  s.homepage    = "http://github.com/SupersonicAds/tailf2kafka"
+  s.summary     = "Watch and tail files with specified time based patterns and push them to kafka"
+  s.description = "Watch and tail files with specified time based patterns and push them to kafka"
+  s.license     = 'MIT'
+  s.has_rdoc    = false 
+
+  s.add_dependency('poseidon')
+  s.add_dependency('hash_symbolizer')
+  s.add_dependency('schash')
+  s.add_dependency('rb-inotify')
+  s.add_dependency('timers')
+
+  s.add_development_dependency('rake')
+
+  s.files         = Dir.glob("{bin,lib}/**/*") + %w(tailf2kafka.gemspec LICENSE README.md)
+  s.executables   = Dir.glob('bin/**/*').map { |file| File.basename(file) }
+  s.test_files    = nil
+  s.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: tailf2kafka
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alexander Piavlo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hash_symbolizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: schash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-inotify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Watch and tail files with specified time based patterns and push them
+  to kafka
+email:
+- devops@supersonic.com
+executables:
+- tailf2kafka
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/tailf2kafka
+- lib/tailf2kafka.rb
+- lib/tailf2kafka/version.rb
+- tailf2kafka.gemspec
+homepage: http://github.com/SupersonicAds/tailf2kafka
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Watch and tail files with specified time based patterns and push them to
+  kafka
+test_files: []