anschel 0.6.5 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bbe6eb2c7595931308ffa51a7c5c246d70bc0bd2
-  data.tar.gz: c982d14ef5c18395f19dea1803875ee81dce3377
+  metadata.gz: 4d2323c2b9351e3241a9f57d0a0d758b1a191748
+  data.tar.gz: 121cbb082eacb6052e330c009ee7e8ca2a553bf0
 SHA512:
-  metadata.gz: a7664952bdf55874f8f883dead99b346034d565462d11e37246ce9437f0d730721eb1726333e492731f159bae02779c5cdbbeaeea80655135fe51ef7a49b6841
-  data.tar.gz: 9dccba8d5cd9cef7ce7e841c95a224807a297091503a9eef98787e75f9d22fbd751149638bca0e0af93ac6d4b4d2f77bb23ec37a0186bfc389e10e9f5bd2f083
+  metadata.gz: 22726febef094ca0aed6f6e2f88d0b101835c165049207b5c0b7c48b1a54f6114158b25db51d8fe7424d4b1d0481bf7cd0f88b33bd3917c7ce519c9ef0db6ca2
+  data.tar.gz: 5e8e436502922f62657918e5c5ae546e12869866e4dcce25a56aa753ea4c51a2319b38909ccb2093575e0dd11ce37752279eee15a283f94e3ce54dd43d90d9ae

data/VERSION

@@ -1 +1 @@
-0.6.5
+0.7.0

data/lib/anschel/filter/convert.rb

@@ -6,7 +6,7 @@
 # }
 module Anschel
   class Filter
-    def convert conf, log
+    def convert conf, stats, log
       field = conf.delete :field
       type  = conf.delete :type
 
@@ -21,12 +21,22 @@ module Anschel
         'string'  => :to_s
       }
 
+      stats.create 'filter-convert'
+      stats.get 'filter-convert'
+      stats.create 'filter-convert-skipped'
+      stats.get 'filter-convert-skippped'
+
       log.trace event: 'filter-compiled', kind: 'convert', \
         field: field, type: type
 
       lambda do |event|
-        return event unless event.has_key? field
+        unless event.has_key? field
+          stats.inc 'filter-convert-skipped'
+          return event
+        end
+
         event[field] = event[field].send type_conversions[type]
+        stats.inc 'filter-convert'
         filtered event, conf
       end
     end

data/lib/anschel/filter/gsub.rb

@@ -7,7 +7,7 @@
 # }
 module Anschel
   class Filter
-    def gsub conf, log
+    def gsub conf, stats, log
       field   = conf.delete :field
       match   = Regexp.new conf.delete(:match)
       replace = conf.delete :replace
@@ -18,13 +18,22 @@ module Anschel
 
       field = field.to_sym
 
+      stats.create 'filter-gsub'
+      stats.get 'filter-gsub'
+      stats.create 'filter-gsub-skipped'
+      stats.get 'filter-gsub-skipped'
 
       log.trace event: 'filter-compiled', kind: 'gsub', \
         field: field, match: match, replace: replace
 
       lambda do |event|
-        return event unless event.has_key? field
+        unless event.has_key? field
+          stats.inc 'filter-gsub-skipped'
+          return event
+        end
+
         event[field].gsub! match, replace
+        stats.inc 'filter-gsub'
         filtered event, conf
       end
     end

data/lib/anschel/filter/index.rb

@@ -8,7 +8,7 @@
 # }
 module Anschel
   class Filter
-    def index conf, log
+    def index conf, stats, log
       stamp  = conf.delete(:stamp)  || '@timestamp'
       prefix = conf.delete(:prefix) || 'logs-%{type}-'
       suffix = conf.delete(:suffix) || '%Y.%m.%d'
@@ -22,30 +22,48 @@ module Anschel
       joda = joda.withDefaultYear(Time.new.year)
       joda = joda.withOffsetParsed
 
+      stats.create 'filter-index'
+      stats.get 'filter-index'
+      stats.create 'filter-index-skipped'
+      stats.get 'filter-index-skipped'
+      stats.create 'filter-index-error'
+      stats.get 'filter-index-error'
 
       log.trace event: 'filter-compiled', kind: 'index', \
         stamp: stamp, prefix: prefix, suffix: suffix, format: format
 
       lambda do |event|
-        return event unless event.has_key? stamp
+        unless event.has_key? stamp
+          stats.inc 'filter-index-skipped'
+          return event
+        end
+
         idx_prefix = prefix % event
+
         begin
           millis     = joda.parseMillis event[stamp]
           idx_suffix = Time.at(0.001 * millis).strftime(suffix)
           event[:_index] = idx_prefix + idx_suffix
+          stats.inc 'filter-index'
           filtered event, conf
         rescue java.lang.IllegalArgumentException => e
           event[:_index] = idx_prefix + Time.now.strftime(suffix)
-          log.warn \
-            event: 'filter-index-warning',
+          log.trace \
+            event: 'filter-index-error',
             reason: 'could not parse event',
             remediation: 'added bogus index',
             remediation: "sending to best-guess index '#{event[:_index]}'",
+            stamp: stamp,
+            prefix: prefix,
+            suffix: suffix,
+            format: format,
             raw_event: event
           if error_tag
             event[:tags] ||= []
             event[:tags] << error_tag
           end
+          stats.inc 'filter-index-error'
+          stats.inc 'filter-index'
           filtered event, conf
         end
       end

data/lib/anschel/filter/parse.rb

@@ -6,7 +6,7 @@
 # }
 module Anschel
   class Filter
-    def parse conf, log
+    def parse conf, stats, log
       field   = conf.delete :field
       pattern = Regexp.new conf.delete(:pattern)
 
@@ -15,25 +15,36 @@ module Anschel
 
       field = field.to_sym
 
+      stats.create 'filter-parse'
+      stats.get 'filter-parse'
+      stats.create 'filter-parse-skipped'
+      stats.get 'filter-parse-skipped'
+      stats.create 'filter-parse-error'
+      stats.get 'filter-parse-error'
 
       log.trace event: 'filter-compiled', kind: 'parse', \
         field: field, pattern: pattern
 
       lambda do |event|
-        return event unless event.has_key? field
+        unless event.has_key? field
+          stats.inc 'filter-parse-skipped'
+          return event
+        end
         mdata = pattern.match event[field]
         if mdata.nil?
-          log.error \
+          log.trace \
             event: 'parse-filter-error',
             reason: 'regexp did not match',
             field: field,
             pattern: pattern,
             raw_event: event
-          return filtered(event, conf)
+          stats.inc 'filter-parse-error'
+          return event
         end
         mdata.names.each do |group|
           event[group.to_sym] = mdata[group]
         end
+        stats.inc 'filter-parse'
         filtered event, conf
       end
     end

data/lib/anschel/filter/scan.rb

@@ -7,7 +7,7 @@
 # }
 module Anschel
   class Filter
-    def scan conf, log
+    def scan conf, stats, log
       field   = conf.delete :field
       pattern = Regexp.new conf.delete(:pattern)
       target  = conf.delete :target
@@ -19,19 +19,44 @@ module Anschel
       field  = field.to_sym
       target = target.to_sym
 
+      stats.create 'filter-scan'
+      stats.get 'filter-scan'
+      stats.create 'filter-scan-skipped'
+      stats.get 'filter-scan-skipped'
+      stats.create 'filter-scan-nomatch'
+      stats.get 'filter-scan-nomatch'
+      stats.create 'filter-scan-error'
+      stats.get 'filter-scan-error'
 
       log.trace event: 'filter-compiled', kind: 'scan', \
         field: field, pattern: pattern, target: target
 
       lambda do |event|
-        return event unless event.has_key? field
-        results = event[field].scan(pattern).flatten.uniq
+        unless event.has_key? field
+          stats.inc 'filter-scan-skipped'
+          return event
+        end
+        begin
+          results = event[field].scan(pattern).flatten.uniq
+        rescue StandardError
+          log.trace \
+            event: 'scan-filter-error',
+            reason: 'could not scan event',
+            field: field,
+            pattern: pattern,
+            target: target,
+            raw_event: event
+          stats.inc 'filter-scan-error'
+          return event
+        end
 
         if results.empty?
+          stats.inc 'filter-scan-nomatch'
           event
         else
           event[target] ||= []
           event[target]  += results
+          stats.inc 'filter-scan'
           filtered event, conf
         end
       end

data/lib/anschel/filter/stamp.rb

@@ -8,7 +8,7 @@
 # }
 module Anschel
   class Filter
-    def stamp conf, log
+    def stamp conf, stats, log
       utc       = conf.delete :utc?
       field     = conf.delete :field
       pattern   = conf.delete :pattern
@@ -32,31 +32,46 @@ module Anschel
 
       offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
 
+      stats.create 'filter-stamp'
+      stats.get 'filter-stamp'
+      stats.create 'filter-stamp-skipped'
+      stats.get 'filter-stamp-skipped'
+      stats.create 'filter-stamp-error'
+      stats.get 'filter-stamp-error'
 
       log.trace event: 'filter-compiled', kind: 'stamp', \
         utc?: utc, field: field, pattern: pattern, target: target
 
       lambda do |event|
-        return event unless event.has_key? field
+        unless event.has_key? field
+          stats.inc 'filter-stamp-skipped'
+          return event
+        end
         parsers.each do |joda|
           begin
             millis = joda.parseMillis event[field]
             event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
+            stats.inc 'filter-stamp'
             return filtered(event, conf)
           rescue
           end
         end
 
-        log.warn \
-          event: 'stamp-filter-warning',
+        log.trace \
+          event: 'stamp-filter-error',
           reason: 'could not parse event',
           remediation: 'using current time for stamp',
+          utc?: utc,
+          field: field,
+          pattern: pattern,
+          target: target,
           raw_event: event
         if error_tag
           event[:tags] ||= []
           event[:tags] << error_tag
         end
         event[target] = Time.now.utc.iso8601(3)
+        stats.inc 'filter-stamp-error'
         filtered event, conf
       end
     end

data/lib/anschel/filter.rb

@@ -20,7 +20,7 @@ module Anschel
         filter_defns.each do |filter_defn|
           filter_type = filter_defn.keys.first
           filter_conf = filter_defn[filter_type]
-          @filters[event_type] << self.send(filter_type, filter_conf, log)
+          @filters[event_type] << self.send(filter_type, filter_conf, stats, log)
         end
       end
       log.info event: 'filter-fully-loaded'

data/lib/anschel/input/rabbitmq.rb

@@ -7,6 +7,8 @@ module Anschel
   class Input
     class RabbitMQ < Base
       def initialize output, config, stats, log
+        connection_defaults = {}
+
         exchange_defaults = {
           type: 'x-consistent-hash',
           durable: true
@@ -23,25 +25,31 @@ module Anschel
           ack: true
         }
 
+        connection = ::MarchHare.connect \
+          connection_defaults.merge(config[:connection] || {})
+
         exchange_name = config[:exchange].delete(:name)
 
         @threads = config[:queues].map do |queue_name, queue_config|
           Thread.new do
-            connection = ::MarchHare.connect config[:connection]
-
             channel = connection.create_channel
 
             exchange = channel.exchange exchange_name, \
               exchange_defaults.merge(config[:exchange])
 
+            subscription = subscription_defaults.merge \
+              (config[:subscription] || {})
+
             queue = channel.queue queue_name.to_s, \
               queue_defaults.merge(queue_config)
 
-            subscription = subscription_defaults.merge \
-              (config[:subscription] || {})
+            log.debug \
+              event: 'input-rabbitmq-connecting-queue',
+              queue: queue_name
 
             queue.subscribe(subscription) do |meta, message|
               output << message
+              stats.inc 'input'
               channel.ack meta.delivery_tag, false if subscription[:ack]
             end
           end
@@ -50,7 +58,7 @@ module Anschel
 
       def stop
         return if @stopped
-        @threads.map &:kill
+        @threads.map(&:kill)
         @stopped = true
       end
     end

data/lib/anschel/input.rb

@@ -8,7 +8,7 @@ module Anschel
       log.info event: 'output-loading'
       log.debug event: 'output-config', config: config, qsize: qsize
 
-      @queue = SizedQueue.new qsize || 2000
+      @queue = SizedQueue.new(qsize || 2000)
 
       Thread.new do
         leftovers ||= []
@@ -18,6 +18,9 @@ module Anschel
 
       @inputs = []
 
+      stats.create 'input'
+      stats.get 'input'
+
       config.each do |input|
         case input.delete(:kind)
         when 'kafka'

data/lib/anschel/main.rb

@@ -85,7 +85,7 @@ module Anschel
           exception: e.inspect,
           class: e.class,
           message: e.message,
-          backtrace: e.backtrace.join("\n")
+          backtrace: e.backtrace
         bye output, input, store, log, :error
         exit 2
       end

data/lib/anschel/mjolnir.rb

@@ -42,7 +42,11 @@ module Anschel
       # Construct a Logger given the command-line options
       def log
         return @logger if defined? @logger
-        @logger = Slog.new out: (options.log || $stdout), prettify: false
+        device = options.log || $stdout
+        colorize, prettify = false, false
+        colorize, prettify = true, true if device.tty? rescue false
+        @logger = Slog.new \
+          out: device, colorize: colorize, prettify: prettify
         @logger.level = :debug if options.debug?
         @logger.level = :trace if options.trace?
         @logger

data/lib/anschel/output/elasticsearch.rb

@@ -50,6 +50,7 @@ module Anschel
             end
 
             client.bulk body: body
+            stats.inc 'output', body.size
           end
         end
       end

data/lib/anschel/output.rb

@@ -10,6 +10,9 @@ module Anschel
 
       @outputs = []
 
+      stats.create 'output'
+      stats.get 'output'
+
       config.each do |output|
         case output.delete(:kind)
         when 'device'

data/lib/anschel/stats.rb

@@ -78,7 +78,7 @@ module Anschel
           stats[k][:val] = stats[k].default
           [
             [ k, v ],
-            [ "#{k}_rate", 1.0 * v / @interval ]
+            [ "#{k}-rate", 1.0 * v / @interval ]
           ]
         end
       end

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.6.5
+  version: 0.7.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-08 00:00:00.000000000 Z
+date: 2015-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
   name: thor
@@ -21,13 +21,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0.pre.java
   name: jruby-kafka
@@ -35,27 +35,27 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0.pre.java
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
   name: jrjackson
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: elasticsearch
@@ -63,27 +63,27 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   name: typhoeus
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
   name: slog
@@ -91,23 +91,23 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.12'
+        version: '2.13'
   name: march_hare
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.12'
+        version: '2.13'
 description: Logstash-like for moving events from Kafka into Elasticsearch.
 email: sczizzo@gmail.com
 executables:
@@ -150,19 +150,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Logstash-like for moving events from Kafka into Elasticsearch
 test_files: []
-has_rdoc: