anschel 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d2323c2b9351e3241a9f57d0a0d758b1a191748
-  data.tar.gz: 121cbb082eacb6052e330c009ee7e8ca2a553bf0
+  metadata.gz: bde923168a56dffdc4774f30fef3c1d7378321dd
+  data.tar.gz: c86137acfc087e9978f16f3547b43b5556785a98
 SHA512:
-  metadata.gz: 22726febef094ca0aed6f6e2f88d0b101835c165049207b5c0b7c48b1a54f6114158b25db51d8fe7424d4b1d0481bf7cd0f88b33bd3917c7ce519c9ef0db6ca2
-  data.tar.gz: 5e8e436502922f62657918e5c5ae546e12869866e4dcce25a56aa753ea4c51a2319b38909ccb2093575e0dd11ce37752279eee15a283f94e3ce54dd43d90d9ae
+  metadata.gz: 84cc50d7e8ab03a37fc9288b2e66e1d1c3068ac5f44576d91aab3a33855c047335e72ccdc8261b82ae5b4a72c04149274e0cead247a947c5956aac96593cfdcf
+  data.tar.gz: 9998968409c6de797d2cebf4aba524bb74f756442ba9ccf36ee1cd71a09f3cfb10d834c0e93c444545bda81565d9addad395e9dbb4c5fa893ec75fc57e084ed5

data/VERSION

@@ -1 +1 @@
-0.7.0
+0.7.1

data/lib/anschel/filter/convert.rb

@@ -22,13 +22,13 @@ module Anschel
       }
 
       stats.create 'filter-convert'
-      stats.get 'filter-convert'
       stats.create 'filter-convert-skipped'
-      stats.get 'filter-convert-skippped'
 
       log.trace event: 'filter-compiled', kind: 'convert', \
         field: field, type: type
 
+
+
       lambda do |event|
         unless event.has_key? field
           stats.inc 'filter-convert-skipped'
@@ -39,6 +39,7 @@ module Anschel
         stats.inc 'filter-convert'
         filtered event, conf
       end
+
     end
   end
 end

data/lib/anschel/filter/debug.rb

@@ -0,0 +1,21 @@
+# {
+#   "debug": {}
+# }
+module Anschel
+  class Filter
+    def debug conf, stats, log
+      log.trace event: 'filter-compiled', kind: 'debug'
+
+
+
+      lambda do |event|
+        log.debug \
+          event: 'debug',
+          event_repr: event.inspect,
+          raw_event: event
+        event
+      end
+
+    end
+  end
+end

data/lib/anschel/filter/gsub.rb

@@ -19,13 +19,13 @@ module Anschel
       field = field.to_sym
 
       stats.create 'filter-gsub'
-      stats.get 'filter-gsub'
       stats.create 'filter-gsub-skipped'
-      stats.get 'filter-gsub-skipped'
 
       log.trace event: 'filter-compiled', kind: 'gsub', \
         field: field, match: match, replace: replace
 
+
+
       lambda do |event|
         unless event.has_key? field
           stats.inc 'filter-gsub-skipped'
@@ -36,6 +36,7 @@ module Anschel
         stats.inc 'filter-gsub'
         filtered event, conf
       end
+
     end
   end
 end

data/lib/anschel/filter/index.rb

@@ -12,26 +12,33 @@ module Anschel
       stamp  = conf.delete(:stamp)  || '@timestamp'
       prefix = conf.delete(:prefix) || 'logs-%{type}-'
       suffix = conf.delete(:suffix) || '%Y.%m.%d'
-      format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" # ISO8601
+      format = conf.delete(:format) || %w[
+        yyyy-MM-dd'T'HH:mm:ss.SSSZZ
+        yyyy-MM-dd'T'HH:mm:ss.SSSZ
+        yyyy-MM-dd'T'HH:mm:ssZZ
+        yyyy-MM-dd'T'HH:mm:ssZ
+      ] # ISO8601
 
       error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'index-error'
 
       stamp = stamp.to_sym
 
-      joda = org.joda.time.format.DateTimeFormat.forPattern format
-      joda = joda.withDefaultYear(Time.new.year)
-      joda = joda.withOffsetParsed
+      format = [ format ] unless format.is_a? Array
+
+      joda = format.map do |f|
+        j = org.joda.time.format.DateTimeFormat.forPattern f
+        j.withDefaultYear(Time.new.year).withOffsetParsed
+      end
 
       stats.create 'filter-index'
-      stats.get 'filter-index'
       stats.create 'filter-index-skipped'
-      stats.get 'filter-index-skipped'
       stats.create 'filter-index-error'
-      stats.get 'filter-index-error'
 
       log.trace event: 'filter-compiled', kind: 'index', \
         stamp: stamp, prefix: prefix, suffix: suffix, format: format
 
+
+
       lambda do |event|
         unless event.has_key? stamp
           stats.inc 'filter-index-skipped'
@@ -40,33 +47,45 @@ module Anschel
 
         idx_prefix = prefix % event
 
-        begin
-          millis     = joda.parseMillis event[stamp]
-          idx_suffix = Time.at(0.001 * millis).strftime(suffix)
-          event[:_index] = idx_prefix + idx_suffix
-          stats.inc 'filter-index'
-          filtered event, conf
-        rescue java.lang.IllegalArgumentException => e
-          event[:_index] = idx_prefix + Time.now.strftime(suffix)
-          log.trace \
-            event: 'filter-index-error',
-            reason: 'could not parse event',
-            remediation: 'added bogus index',
-            remediation: "sending to best-guess index '#{event[:_index]}'",
-            stamp: stamp,
-            prefix: prefix,
-            suffix: suffix,
-            format: format,
-            raw_event: event
-          if error_tag
-            event[:tags] ||= []
-            event[:tags] << error_tag
+        matched = false
+
+        joda.each do |j|
+          begin
+            millis = j.parseMillis event[stamp]
+            idx_suffix = Time.at(0.001 * millis).utc.strftime(suffix)
+            event[:_index] = idx_prefix + idx_suffix
+            stats.inc 'filter-index'
+            matched = true
+          rescue java.lang.IllegalArgumentException => e
           end
-          stats.inc 'filter-index-error'
-          stats.inc 'filter-index'
-          filtered event, conf
+          break if matched
+        end
+
+        return filtered(event, conf) if matched
+
+        event[:_index] = idx_prefix + Time.now.utc.strftime(suffix)
+
+        log.warn \
+          event: 'filter-index-warning',
+          reason: 'could not parse event',
+          remediation: 'added bogus index',
+          remediation: "sending to best-guess index '#{event[:_index]}'",
+          stamp: stamp,
+          prefix: prefix,
+          suffix: suffix,
+          format: format,
+          raw_event: event.inspect
+
+        if error_tag
+          event[:tags] ||= []
+          event[:tags] << error_tag
         end
+
+        stats.inc 'filter-index-error'
+        stats.inc 'filter-index'
+        filtered event, conf
       end
+
     end
   end
 end

data/lib/anschel/filter/parse.rb

@@ -7,30 +7,40 @@
 module Anschel
   class Filter
     def parse conf, stats, log
-      field   = conf.delete :field
+      field = conf.delete :field
       pattern = Regexp.new conf.delete(:pattern)
+      unless_field = conf.delete(:unless_field) || '@timestamp'
+
+      error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'parse-error'
 
       raise 'Missing required "field" for "parse" filter' if field.nil?
       raise 'Missing required "pattern" for "parse" filter' if pattern.nil?
 
       field = field.to_sym
+      unless_field = unless_field.to_sym
 
       stats.create 'filter-parse'
-      stats.get 'filter-parse'
       stats.create 'filter-parse-skipped'
-      stats.get 'filter-parse-skipped'
       stats.create 'filter-parse-error'
-      stats.get 'filter-parse-error'
 
       log.trace event: 'filter-compiled', kind: 'parse', \
         field: field, pattern: pattern
 
+
+
       lambda do |event|
         unless event.has_key? field
           stats.inc 'filter-parse-skipped'
           return event
         end
+
+        if event.has_key? unless_field
+          stats.inc 'filter-parse-skipped'
+          return event
+        end
+
         mdata = pattern.match event[field]
+
         if mdata.nil?
           log.trace \
             event: 'parse-filter-error',
@@ -39,14 +49,21 @@ module Anschel
             pattern: pattern,
             raw_event: event
           stats.inc 'filter-parse-error'
+          if error_tag
+            event[:tags] ||= []
+            event[:tags] << error_tag
+          end
           return event
         end
+
         mdata.names.each do |group|
           event[group.to_sym] = mdata[group]
         end
+
         stats.inc 'filter-parse'
         filtered event, conf
       end
+
     end
   end
 end

data/lib/anschel/filter/scan.rb

@@ -9,37 +9,52 @@ module Anschel
   class Filter
     def scan conf, stats, log
       field   = conf.delete :field
-      pattern = Regexp.new conf.delete(:pattern)
+      pattern = conf.delete :pattern
       target  = conf.delete :target
 
+      error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'scan-error'
+
       raise 'Missing required "field" for "scan" filter' if field.nil?
       raise 'Missing required "pattern" for "scan" filter' if pattern.nil?
       raise 'Missing required "target" for "convert" filter' if target.nil?
 
       field  = field.to_sym
       target = target.to_sym
+      match  = Regexp.new pattern
 
       stats.create 'filter-scan'
-      stats.get 'filter-scan'
       stats.create 'filter-scan-skipped'
-      stats.get 'filter-scan-skipped'
       stats.create 'filter-scan-nomatch'
-      stats.get 'filter-scan-nomatch'
       stats.create 'filter-scan-error'
-      stats.get 'filter-scan-error'
 
       log.trace event: 'filter-compiled', kind: 'scan', \
-        field: field, pattern: pattern, target: target
+        field: field, pattern: pattern, match: match, target: target
+
+
 
       lambda do |event|
         unless event.has_key? field
           stats.inc 'filter-scan-skipped'
           return event
         end
+
+        error = true
         begin
-          results = event[field].scan(pattern).flatten.uniq
+          results = event[field].scan(match).flatten.uniq.map do |s|
+            s.reverse.reverse # N.B. There seems to be some issue with the "scan"
+                              #      function in JRuby wherein the matches are
+                              #      shared across threads or somehow mangled.
+                              #      The reverse.reverse here ensures that we
+                              #      create a new object with the original
+                              #      contents still intact. If you have a
+                              #      better solution, please contact me!
+          end
+          error   = false
         rescue StandardError
-          log.trace \
+        end
+
+        if error
+          log.error \
             event: 'scan-filter-error',
             reason: 'could not scan event',
             field: field,
@@ -47,12 +62,16 @@ module Anschel
             target: target,
             raw_event: event
           stats.inc 'filter-scan-error'
-          return event
-        end
+          if error_tag
+            event[:tags] ||= []
+            event[:tags] << error_tag
+          end
+          event
 
-        if results.empty?
+        elsif results.empty?
           stats.inc 'filter-scan-nomatch'
           event
+
         else
           event[target] ||= []
           event[target]  += results
@@ -60,6 +79,7 @@ module Anschel
           filtered event, conf
         end
       end
+
     end
   end
 end

data/lib/anschel/filter/stamp.rb

@@ -6,6 +6,9 @@
 #     "target": "@timestamp"
 #   }
 # }
+require 'date'
+require 'time'
+
 module Anschel
   class Filter
     def stamp conf, stats, log
@@ -13,6 +16,8 @@ module Anschel
       field     = conf.delete :field
       pattern   = conf.delete :pattern
       target    = conf.delete :target
+      precision = conf.delete(:precision) || 3
+
       error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'stamp-error'
 
       raise 'Missing required "field" for "stamp" filter' if field.nil?
@@ -33,32 +38,52 @@ module Anschel
       offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
 
       stats.create 'filter-stamp'
-      stats.get 'filter-stamp'
       stats.create 'filter-stamp-skipped'
-      stats.get 'filter-stamp-skipped'
       stats.create 'filter-stamp-error'
-      stats.get 'filter-stamp-error'
 
       log.trace event: 'filter-compiled', kind: 'stamp', \
         utc?: utc, field: field, pattern: pattern, target: target
 
+
+
       lambda do |event|
         unless event.has_key? field
           stats.inc 'filter-stamp-skipped'
           return event
         end
+
+        if event.has_key? target
+          log.warn \
+            event: 'stamp-filter-warning',
+            reason: 'event already has target field',
+            utc?: utc,
+            field: field,
+            pattern: pattern,
+            target: target,
+            raw_event: event
+          event[target] = \
+            DateTime.parse(event[target]).to_time.utc.iso8601(precision)
+          return event
+        end
+
+        event_field = event[field].dup
+
+        matched = false
         parsers.each do |joda|
           begin
             millis = joda.parseMillis event[field]
-            event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
+            event[target] = Time.at(0.001 * millis + offset_s).utc.iso8601(precision)
             stats.inc 'filter-stamp'
-            return filtered(event, conf)
+            matched = true
           rescue
           end
+          break if matched
         end
 
-        log.trace \
-          event: 'stamp-filter-error',
+        return filtered(event, conf) if matched
+
+        log.warn \
+          event: 'stamp-filter-warning',
           reason: 'could not parse event',
           remediation: 'using current time for stamp',
           utc?: utc,
@@ -66,14 +91,17 @@ module Anschel
           pattern: pattern,
           target: target,
           raw_event: event
+
         if error_tag
           event[:tags] ||= []
           event[:tags] << error_tag
         end
-        event[target] = Time.now.utc.iso8601(3)
+
+        event[target] = Time.now.utc.iso8601(precision)
         stats.inc 'filter-stamp-error'
         filtered event, conf
       end
+
     end
   end
 end

data/lib/anschel/filter/tag.rb

@@ -0,0 +1,31 @@
+# {
+#   "tag": {
+#     "with": [ "" ]
+#   }
+# }
+module Anschel
+  class Filter
+    def tag conf, stats, log
+      tags = conf.delete :with
+
+      raise 'Missing required "with" for "tag" filter' if tags.nil?
+
+      tags = tags.is_a?(Array) ? tags : [ tags ]
+
+      stats.create 'filter-tag'
+
+      log.trace event: 'filter-compiled', kind: 'tag', with: tags
+
+
+
+      lambda do |event|
+        event[:tags] ||= []
+        event[:tags]  += tags
+        event[:tags].uniq!
+        stats.inc 'filter-tag'
+        filtered event, conf
+      end
+
+    end
+  end
+end

data/lib/anschel/filter.rb

@@ -4,6 +4,8 @@ require_relative 'filter/index'
 require_relative 'filter/parse'
 require_relative 'filter/scan'
 require_relative 'filter/stamp'
+require_relative 'filter/tag'
+require_relative 'filter/debug'
 
 
 module Anschel

data/lib/anschel/input.rb

@@ -12,7 +12,7 @@ module Anschel
 
       Thread.new do
         leftovers ||= []
-        log.trace event: 'input-leftovers', leftovers_size: leftovers.size
+        log.warn event: 'input-leftovers', leftovers_size: leftovers.size
         leftovers.each { |l| @queue << l }
       end
 
@@ -25,10 +25,10 @@ module Anschel
         case input.delete(:kind)
         when 'kafka'
           @inputs << Input::Kafka.new(@queue, input, stats, log)
-          log.trace event: 'input-loaded', kind: 'kafka'
+          log.debug event: 'input-loaded', kind: 'kafka'
         when 'rabbitmq'
           @inputs << Input::RabbitMQ.new(@queue, input, stats, log)
-          log.trace event: 'input-loaded', kind: 'rabbitmq'
+          log.debug event: 'input-loaded', kind: 'rabbitmq'
         else
           raise 'Uknown input type'
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -122,11 +122,13 @@ files:
 - lib/anschel.rb
 - lib/anschel/filter.rb
 - lib/anschel/filter/convert.rb
+- lib/anschel/filter/debug.rb
 - lib/anschel/filter/gsub.rb
 - lib/anschel/filter/index.rb
 - lib/anschel/filter/parse.rb
 - lib/anschel/filter/scan.rb
 - lib/anschel/filter/stamp.rb
+- lib/anschel/filter/tag.rb
 - lib/anschel/input.rb
 - lib/anschel/input/base.rb
 - lib/anschel/input/kafka.rb