anschel 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f81ae5fdcf71f580680520067255911285629aa
-  data.tar.gz: 68ab5e077fa13d9d4e142fe798f222912dce5f6a
+  metadata.gz: 70986c1daf98761ecae7d688a9cac9dca77d5245
+  data.tar.gz: 423be3728f6d0b12816965df20cb48c3ccc77198
 SHA512:
-  metadata.gz: 0179da5dc7e53fc18e3c490f4e49f521e210184a0df80a5d834052237b2f476f4b99fbcad40bd390f1f318a9af9559f9609117098d8e7cf02d35829a560d2aab
-  data.tar.gz: 95309336805570a5e7cd50dfa1d47e43677caa3a72f78ea1a8e8b7fadc68146c26e00feb73706b0a2620868f73777af78b94c4b6368eeb8f4bc1c8e30c74d88f
+  metadata.gz: 82ff4b34d365f33a3a39cc3bac477cac4c8c449ebfadb8317a15a92372bc6af858ee2715afd6329d976fa70e2f07e7c4cff14f0ccf6a14a61cefb30d1a748052
+  data.tar.gz: db692abab264dc1ad1ccee814560071e398d81430717e2b61d76469e83754128bf0277820eadc55231e131719271d79851aa4bff51902bf7372ca40f08c1c2a2

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.6.2

data/lib/anschel/filter/convert.rb

@@ -7,7 +7,6 @@
 module Anschel
   class Filter
     def convert conf, log
-      log.debug event: 'compile-filter', filter: 'convert', conf: conf
       field = conf.delete :field
       type  = conf.delete :type
 
@@ -22,7 +21,7 @@ module Anschel
         'string'  => :to_s
       }
 
-      log.debug event: 'compiled-filter', filter: 'convert', \
+      log.trace event: 'filter-compiled', kind: 'convert', \
         field: field, type: type
 
       lambda do |event|

data/lib/anschel/filter/gsub.rb

@@ -19,7 +19,7 @@ module Anschel
       field = field.to_sym
 
 
-      log.debug event: 'compiled-filter', filter: 'gsub', \
+      log.trace event: 'filter-compiled', kind: 'gsub', \
         field: field, match: match, replace: replace
 
       lambda do |event|

data/lib/anschel/filter/index.rb

@@ -23,7 +23,7 @@ module Anschel
       joda = joda.withOffsetParsed
 
 
-      log.debug event: 'compiled-filter', filter: 'index', \
+      log.trace event: 'filter-compiled', kind: 'index', \
         stamp: stamp, prefix: prefix, suffix: suffix, format: format
 
       lambda do |event|

data/lib/anschel/filter/parse.rb

@@ -16,7 +16,7 @@ module Anschel
       field = field.to_sym
 
 
-      log.debug event: 'compiled-filter', filter: 'parse', \
+      log.trace event: 'filter-compiled', kind: 'parse', \
         field: field, pattern: pattern
 
       lambda do |event|

data/lib/anschel/filter/scan.rb

@@ -20,7 +20,7 @@ module Anschel
       target = target.to_sym
 
 
-      log.debug event: 'compiled-filter', filter: 'scan', \
+      log.trace event: 'filter-compiled', kind: 'scan', \
         field: field, pattern: pattern, target: target
 
       lambda do |event|

data/lib/anschel/filter/stamp.rb

@@ -2,7 +2,8 @@
 #   "stamp": {
 #     "utc?": false,
 #     "field": "timestamp",
-#     "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
+#     "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ],
+#     "target": "@timestamp"
 #   }
 # }
 module Anschel
@@ -31,8 +32,9 @@ module Anschel
 
       offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
 
-      log.debug event: 'compiled-filter', filter: 'stamp', \
-        field: field, pattern: pattern, target: target, error_tag: error_tag
+
+      log.trace event: 'filter-compiled', kind: 'stamp', \
+        utc?: utc, field: field, pattern: pattern, target: target
 
       lambda do |event|
         return event unless event.has_key? field

data/lib/anschel/filter.rb

@@ -12,7 +12,8 @@ module Anschel
     attr_reader :filters
 
     def initialize config, stats, log
-      log.trace event: 'filter', config: config
+      log.info event: 'filter-loading'
+      log.debug event: 'filter-config', config: config
       config ||= {} # Allow for nil config
       @filters = Hash.new { |h,k| h[k] = [] }
       config.each do |event_type, filter_defns|
@@ -22,7 +23,7 @@ module Anschel
           @filters[event_type] << self.send(filter_type, filter_conf, log)
         end
       end
-      log.info event: 'filter-loaded'
+      log.info event: 'filter-fully-loaded'
     end
 
 

data/lib/anschel/input/kafka.rb

@@ -7,7 +7,6 @@ module Anschel
   class Input
     class Kafka < Base
       def initialize queue, config, stats, log
-        log.trace event: 'input', kind: 'kafka', config: config
         @consumer_group = ::Kafka::Group.new config
         @consumer_group.run num_cpus, queue
       end

data/lib/anschel/input/rabbitmq.rb

@@ -7,8 +7,6 @@ module Anschel
   class Input
     class RabbitMQ < Base
       def initialize queue, config, stats, log
-        log.trace event: 'input', kind: 'rabbitmq', config: config
-
         default_exchange = {
           type: 'x-consistent-hash',
           durable: true

data/lib/anschel/input.rb

@@ -4,34 +4,47 @@ require_relative 'input/rabbitmq'
 
 module Anschel
   class Input
-    def initialize config, qsize, stats, log
-      if config.empty?
-        raise 'No input provided'
-      end
+    def initialize config, qsize, stats, log, leftovers=[]
+      log.info event: 'output-loading'
+      log.debug event: 'output-config', config: config, qsize: qsize
 
       @queue = SizedQueue.new qsize || 2000
 
+      Thread.new do
+        leftovers ||= []
+        log.trace event: 'input-leftovers', leftovers_size: leftovers.size
+        leftovers.each { |l| @queue << l }
+      end
+
       @inputs = []
 
       config.each do |input|
         case input.delete(:kind)
         when 'kafka'
           @inputs << Input::Kafka.new(@queue, input, stats, log)
+          log.trace event: 'input-loaded', kind: 'kafka'
         when 'rabbitmq'
           @inputs << Input::RabbitMQ.new(@queue, input, stats, log)
+          log.trace event: 'input-loaded', kind: 'rabbitmq'
         else
           raise 'Uknown input type'
         end
       end
 
-      log.info event: 'input-loaded'
+      log.info event: 'input-fully-loaded'
     end
 
 
     def stop
-      return if @stopped
+      return @leftovers if defined? @leftovers
       @inputs.map &:stop
-      @stopped = true
+      @leftovers = []
+      @leftovers << shift until @queue.empty?
+    end
+
+
+    def leftovers
+      return @leftovers if defined? @leftovers
     end
 
 

data/lib/anschel/main.rb

@@ -5,6 +5,7 @@ require 'jrjackson'
 
 require_relative 'mjolnir'
 require_relative 'metadata'
+require_relative 'store'
 require_relative 'stats'
 require_relative 'input'
 require_relative 'filter'
@@ -46,44 +47,70 @@ module Anschel
         options: options.to_hash,
         num_cpus: num_cpus
 
-      config = JrJackson::Json.load \
-        File.read(options.config), symbolize_keys: true
+      input, filter, output, store, stats, ts = \
+        nil, nil, nil, nil, nil, nil, nil
 
-      setup_log4j config[:log4j]
+      begin
+        config = JrJackson::Json.load \
+          File.read(options.config), symbolize_keys: true
 
-      stats  = Stats.new log, options.stats_interval
+        setup_log4j config[:log4j]
 
-      input  = Input.new config[:input], config[:queue_size], stats, log
+        store  = Store.new config[:store], log
 
-      filter = Filter.new config[:filter], stats, log
+        stats  = Stats.new log, options.stats_interval
 
-      output = Output.new config[:output], stats, log
+        filter = Filter.new config[:filter], stats, log
 
-      trap('SIGINT') do
-        input.stop
-        output.stop
-        log.info event: 'goodbye', version: VERSION
-        exit
-      end
+        output = Output.new config[:output], stats, log
 
+        input  = Input.new \
+          config[:input], config[:queue_size], stats, log, store[:input]
 
-      stats.create 'event'
-      stats.get 'event'
+        stats.create 'event'
+        stats.get 'event'
 
-      ts = num_cpus.times.map do
-        Thread.new do
-          loop do
-            event = JrJackson::Json.load \
-              input.shift, symbolize_keys: true
-            output.push filter.apply(event)
-            stats.inc 'event'
+        ts = num_cpus.times.map do
+          Thread.new do
+            loop do
+              event = JrJackson::Json.load \
+                input.shift, symbolize_keys: true
+              output.push filter.apply(event)
+              stats.inc 'event'
+            end
           end
         end
+
+      rescue => e
+        log.fatal \
+          event: 'exception',
+          exception: e.inspect,
+          class: e.class,
+          message: e.message,
+          backtrace: e.backtrace.join("\n")
+        bye output, input, store, log, :error
+        exit 2
+      end
+
+      log.info event: 'all-systems-clear'
+
+      trap('SIGINT') do
+        bye output, input, store, log
+        exit
       end
 
-      log.info event: 'fully-loaded'
       ts.map &:join
-      exit
+    end
+
+
+  private
+    def bye output, input, store, log, level=:info
+      if input && output
+        output.stop ; input.stop
+        store[:input] = input.leftovers
+        log.debug event: 'flush'
+      end
+      log.send level, event: 'goodbye', version: VERSION
     end
 
   end

data/lib/anschel/output/device.rb

@@ -7,8 +7,6 @@ module Anschel
   class Output
     class Device < Base
       def initialize config, stats, log
-        log.trace event: 'output', kind: 'device', config: config
-
         path   = config.delete(:path) || '/dev/stdout'
         qsize  = config.delete(:queue_size) || 2000
         @queue = SizedQueue.new qsize

data/lib/anschel/output/elasticsearch.rb

@@ -11,8 +11,7 @@ module Anschel
   class Output
     class Elasticsearch < Base
       def initialize config, stats, log
-        log.trace event: 'output', kind: 'elasticsearch', config: config
-        default_index = config.delete(:default_index) || 'logs-anschel'
+        default_index = config.delete(:default_index) || '.anschel'
         qsize   = config.delete(:queue_size) || 2000
         bsize   = config.delete(:bulk_size) || 500
         timeout = config.delete(:bulk_timeout) || 0.5

data/lib/anschel/output.rb

@@ -5,9 +5,8 @@ require_relative 'output/device'
 module Anschel
   class Output
     def initialize config, stats, log
-      if config.empty?
-        raise 'No output provided'
-      end
+      log.info event: 'output-loading'
+      log.debug event: 'output-config', config: config
 
       @outputs = []
 
@@ -15,14 +14,16 @@ module Anschel
         case output.delete(:kind)
         when 'device'
           @outputs << Output::Device.new(output, stats, log)
+          log.trace event: 'output-loaded', kind: 'device'
         when 'elasticsearch'
           @outputs << Output::Elasticsearch.new(output, stats, log)
+          log.trace event: 'output-loaded', kind: 'elasticsearch'
         else
           raise 'Unkown output type'
         end
       end
 
-      log.info event: 'output-loaded'
+      log.info event: 'output-fully-loaded'
     end
 
 
@@ -32,6 +33,7 @@ module Anschel
       @stopped = true
     end
 
+
     def push event
       @outputs.each do |output|
         output.push event

data/lib/anschel/stats.rb

@@ -83,6 +83,7 @@ module Anschel
         end
       end
 
+      return if stats.empty?
       @logger.info \
         event: 'stats',
         interval: @interval,

data/lib/anschel/store.rb

@@ -0,0 +1,27 @@
+require 'jrjackson'
+
+
+module Anschel
+  class Store
+    def initialize path, log
+      @path = path || '/tmp/anschel.db'
+      @hash = nil
+      if File.exist? @path
+        @hash = JrJackson::Json.load \
+          File.read(@path), symbolize_keys: true
+      end
+      @hash ||= {}
+      log.info event: 'stats-loaded'
+    end
+
+    def [] k ; @hash[k] end
+
+    def []= k,v ; @hash[k] = v ; save end
+
+    def save
+      File.open(@path, 'w') do |f|
+        f.puts JrJackson::Json.dump(@hash)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Sean Clemmer
@@ -139,6 +139,7 @@ files:
 - lib/anschel/output/device.rb
 - lib/anschel/output/elasticsearch.rb
 - lib/anschel/stats.rb
+- lib/anschel/store.rb
 homepage: https://github.com/sczizzo/anschel
 licenses:
 - ISC