tailf2kafka 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0511b332f01cad011509e5312d35a44406996394
-  data.tar.gz: 1ca828eed00c80a7896b3cf384292e577d8832f5
+  metadata.gz: c4c3bad1a4caca5027bba89946e2a38de39bb6b7
+  data.tar.gz: 07b8a91f46d3592384200bc07418cfc0c514615f
 SHA512:
-  metadata.gz: bbe8203735d60e6483a5aef2f0a3f6c0473828753fa5d4649313b56a147b96e63fa300bec9f6a5900995c6c56578b69e4db6af88d572c1e946fbb8a687508f56
-  data.tar.gz: 1e9c1d14ad055d267a005208a5a2a59bd9a5786984bda8b323aff08b6703a75c596761ce688c9ec1552d8fd3a15dc6e5b78e1090ab7e9dd740890b937ab85997
+  metadata.gz: 62bba4611e56095855356bfd0e76c2173004f1948dcab31495701b94c29849b60df7a88a940b60334172f09f03b79606a8ee40a1b89518c9dc82e723c7334dae
+  data.tar.gz: 34232db94d928667aded3b4cd6ac544f8a487993087938b74f626806bd0a6e14f9bf3ff2c982242fc2f55955181e50530d16eb387a07924a454812aff6e20640

data/README.md

@@ -5,13 +5,17 @@ Watch and tail files in dirs with specified filename time based patterns and pus
 
 ## Installation
 
+Install libsnappy dev libs if you want to take advantage of compression
+
+    apt-get install libsnappy-dev
+
 Add this line to your application's Gemfile:
 
     gem 'tailf2kafka'
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
@@ -31,6 +35,7 @@ Or install it yourself as:
       files:
         - topic: haproxy
           prefix: /var/log/haproxy/haproxy
+          suffix: ''
           time_pattern: ".%Y-%m-%d.%H"
       position_file: "/var/lib/haproxy/tail2kafka.offsets"
       flush_interval: 1
@@ -40,6 +45,32 @@ Or install it yourself as:
     kafka:
       brokers: ["broker1:9092", "broker2:9092", "broker3:9092"]
       producer_type: sync
+      produce: true
+
+* kafka.brokers - Array of kafka brokers to connect to
+* kafka.producer_type - type of producer sync or async
+* kafka.produce - if false will not conect to kafka and will not produce any messages to it
+* tailf.position_file - file where to save tailed files offsets which were pushed to kafka
+* tailf.flush_interval - how often in seconds to save the offsets to a file
+* tailf.max_batch_lines - max number of lines to batch in each send request
+* tailf.from_beggining - in case of a new file added to tailing , if to start tailing from beggining or end of the file
+* tailf.delete_old_tailed_files - if to delete files once their time_pattern does not match the current time window and if they have been fully produced to kafka
+* tailf.files - array of file configs for tail, each tailed file configs consists of:
+  * topic - which kafka topic to produce the messages to
+  * prefix - the files prefix to watch for
+  * time_pattern - ruby time pattern of files to tail
+  * suffix - optional suffix of files to watch for
+so the tool will watch for files that match - prefix + time_pattern + suffix
+
+## Features/Facts
+
+* The config is validated by [schash](https://github.com/ryotarai/schash) gem
+* Tailed files are watched for changes by [rb-notify](https://github.com/nex3/rb-inotify) gem
+* Dirnames of all files prefixes are watched for new files creation or files moved to the dir and are automaticaly
+added to tailing.
+* As well dirnames are watched for deletion or files being moved out of directory, and they are removed from the  list of files watched for changing.
+* Based time_pattern, files are periodicaly autodeleted , thus avoiding need for log rotation tools.
+* Files are matched by converting time_pattern to a regexp
 
 ## Contributing
 

data/bin/tailf2kafka

@@ -168,7 +168,7 @@ end
   dir = File.dirname(tailf_file[:prefix])
   if File.exists?(dir) and File.directory?(dir)
     @dirs[dir] ||= []
-    @dirs[dir] << { :prefix => File.basename(tailf_file[:prefix]), :pattern => tailf_file[:time_pattern], :suffix => "#{tailf_file[:suffix]}" }
+    @dirs[dir] << { :prefix => File.basename(tailf_file[:prefix]), :pattern => tailf_file[:time_pattern], :suffix => "#{tailf_file[:suffix]}", :topic => tailf_file[:topic]}
     Dir.glob("#{tailf_file[:prefix]}*#{tailf_file[:suffix]}").each do |path|
       if path.match(Regexp.new(time_pattern_to_regexp(tailf_file[:time_pattern])))
         unless File.directory?(path)
@@ -207,12 +207,12 @@ Thread.new { loop { @timers.wait } }
   @create_notifier.watch(dir, :create, :moved_to) do |event|
     @mutex.synchronize do
       path = "#{dir}/#{event.name}"
-      matches = @dirs[dir].select{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}.empty?
-      unless matches.empty?
+      match = @dirs[dir].detect{|h| event.name.match(Regexp.new(h[:prefix] + time_pattern_to_regexp(h[:pattern]) + h[:suffix]))}
+      if match
         unless File.directory?(path)
           unless @threads.has_key?(path)
             puts "File #{event.name} was created in / moved into watched dir #{dir}"
-            @files[path] = { :pattern => match.first[:pattern], :topic => File.basename(@dirs[dir].detect{|prefix| path.start_with?(prefix)}), :inode => File.stat(path).ino, :offset => 0 }
+            @files[path] = { :pattern => match[:pattern], :topic => match[:topic], :inode => File.stat(path).ino, :offset => 0 }
             @threads[path] = Thread.new { tailf(path) }
           end
         end

data/lib/tailf2kafka/version.rb

@@ -1,3 +1,3 @@
 module Tailf2Kafka
-  VERSION ||= '0.1.0'
+  VERSION ||= '0.1.1'
 end

data/tailf2kafka.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
   s.has_rdoc    = false 
 
   s.add_dependency('poseidon')
+  s.add_dependency('snappy')
   s.add_dependency('hash_symbolizer')
   s.add_dependency('schash')
   s.add_dependency('rb-inotify')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tailf2kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Alexander Piavlo
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: hash_symbolizer
   requirement: !ruby/object:Gem::Requirement