anschel 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a2475c658cf220a706e32d1bd87816b1278465d
-  data.tar.gz: b6a9b99ff3ce1b8829373c1abc08809f9b0e7660
+  metadata.gz: 8f81ae5fdcf71f580680520067255911285629aa
+  data.tar.gz: 68ab5e077fa13d9d4e142fe798f222912dce5f6a
 SHA512:
-  metadata.gz: 5e6882bb74eb0f96074c0a126c67eb9151d49ad64ab094fc91089cbdc9e3ae5d8603748b21e3190c6192defea4885d2517f632270dde4ac5a38ac3efc4a65e95
-  data.tar.gz: fd482f04a614e36a9f75a104e77ed52f2905e80730a35aabe485d8c2cce20918283eec6a2e447c9c3ff9b0dc347c41cd19107ae1c1a8020f6d5f5e215ef18f24
+  metadata.gz: 0179da5dc7e53fc18e3c490f4e49f521e210184a0df80a5d834052237b2f476f4b99fbcad40bd390f1f318a9af9559f9609117098d8e7cf02d35829a560d2aab
+  data.tar.gz: 95309336805570a5e7cd50dfa1d47e43677caa3a72f78ea1a8e8b7fadc68146c26e00feb73706b0a2620868f73777af78b94c4b6368eeb8f4bc1c8e30c74d88f

data/Readme.md

@@ -54,10 +54,13 @@ It's kinda like a JSON version of the Logstash config language:
         "pattern": "[%d] %p %m (%c)%n"
       },
 
-      // Kafka is the primary input; see the `jruby-kafka` homepage for
-      // more details: https://github.com/joekiller/jruby-kafka
-      "input": {
-        "kafka": {
+      // Specify any number of inputs
+      "input": [
+
+        // Kafka is the primary input; see the `jruby-kafka` homepage for
+        // more details: https://github.com/joekiller/jruby-kafka
+        {
+          "kind": "kafka",
           "queue_size": 2000,
           "zk_connect": "localhost:2181",
           "zk_connect_timeout": 6000,
@@ -81,12 +84,15 @@ It's kinda like a JSON version of the Logstash config language:
           "consumer_timeout_ms": -1,
           "consumer_restart_sleep_ms": 0
         }
-      },
+      ],
+
+      // Specify any number of outputs
+      "output": [
 
-      // Elasticsearch is the primary output; see the `elasticsearch-ruby`
-      // homepage for more: https://github.com/elastic/elasticsearch-ruby
-      "output": {
-        "elasticsearch": {
+        // Elasticsearch is the primary output; see the `elasticsearch-ruby`
+        // homepage for more: https://github.com/elastic/elasticsearch-ruby
+        {
+          "kind": "elasticsearch",
           "queue_size": 2000,
           "bulk_size": 200,
           "hosts": [ "localhost:9200" ],
@@ -95,7 +101,7 @@ It's kinda like a JSON version of the Logstash config language:
           "reload_on_failure": true,
           "sniffer_timeout": 5
         }
-      },
+      ],
 
       // Just like Logstash, Anschel has a notion of filters
       "filter": {
@@ -148,7 +154,6 @@ You might deploy Anschel with Upstart. Here's a minimal config:
 
     start on startup
     stop on shutdown
-    respawn
 
     exec java -jar anschel-x.y.z.jar \
       --config /etc/anschel.json --log /var/log/anschel.log
@@ -161,7 +166,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
 
 _In development_
 
-- Intial implementation of the Kafka-to-Elasticsearch pipeline
-- Support for file (device) output (v0.4)
+- Allow multiple input and output configurations (v0.6)
 - Support for RabbitMQ input (v0.5)
-- Allow multiple input and output configuration (v0.6)
+- Support for file (device) output (v0.4)
+- Intial implementation of the Kafka-to-Elasticsearch pipeline

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

data/lib/anschel/filter/index.rb

@@ -12,7 +12,9 @@ module Anschel
       stamp  = conf.delete(:stamp)  || '@timestamp'
       prefix = conf.delete(:prefix) || 'logs-%{type}-'
       suffix = conf.delete(:suffix) || '%Y.%m.%d'
-      format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZ" # ISO8601
+      format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" # ISO8601
+
+      error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'index-error'
 
       stamp = stamp.to_sym
 
@@ -26,11 +28,26 @@ module Anschel
 
       lambda do |event|
         return event unless event.has_key? stamp
-        millis     = joda.parseMillis event[stamp]
         idx_prefix = prefix % event
-        idx_suffix = Time.at(0.001 * millis).strftime(suffix)
-        event[:_index] = idx_prefix + idx_suffix
-        filtered event, conf
+        begin
+          millis     = joda.parseMillis event[stamp]
+          idx_suffix = Time.at(0.001 * millis).strftime(suffix)
+          event[:_index] = idx_prefix + idx_suffix
+          filtered event, conf
+        rescue java.lang.IllegalArgumentException => e
+          event[:_index] = idx_prefix + Time.now.strftime(suffix)
+          log.warn \
+            event: 'filter-index-warning',
+            reason: 'could not parse event',
+            remediation: 'added bogus index',
+            remediation: "sending to best-guess index '#{event[:_index]}'",
+            raw_event: event
+          if error_tag
+            event[:tags] ||= []
+            event[:tags] << error_tag
+          end
+          filtered event, conf
+        end
       end
     end
   end

data/lib/anschel/filter/parse.rb

@@ -22,6 +22,15 @@ module Anschel
       lambda do |event|
         return event unless event.has_key? field
         mdata = pattern.match event[field]
+        if mdata.nil?
+          log.error \
+            event: 'parse-filter-error',
+            reason: 'regexp did not match',
+            field: field,
+            pattern: pattern,
+            raw_event: event
+          return filtered(event, conf)
+        end
         mdata.names.each do |group|
           event[group.to_sym] = mdata[group]
         end

data/lib/anschel/filter/stamp.rb

@@ -1,5 +1,6 @@
 # {
 #   "stamp": {
+#     "utc?": false,
 #     "field": "timestamp",
 #     "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
 #   }
@@ -7,6 +8,7 @@
 module Anschel
   class Filter
     def stamp conf, log
+      utc       = conf.delete :utc?
       field     = conf.delete :field
       pattern   = conf.delete :pattern
       target    = conf.delete :target
@@ -27,6 +29,7 @@ module Anschel
         joda = joda.withOffsetParsed
       end
 
+      offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
 
       log.debug event: 'compiled-filter', filter: 'stamp', \
         field: field, pattern: pattern, target: target, error_tag: error_tag
@@ -34,19 +37,25 @@ module Anschel
       lambda do |event|
         return event unless event.has_key? field
         parsers.each do |joda|
-          millis = joda.parseMillis event[field]
           begin
-            event[target] = Time.at(0.001 * millis).iso8601(3)
+            millis = joda.parseMillis event[field]
+            event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
             return filtered(event, conf)
           rescue
           end
         end
 
+        log.warn \
+          event: 'stamp-filter-warning',
+          reason: 'could not parse event',
+          remediation: 'using current time for stamp',
+          raw_event: event
         if error_tag
           event[:tags] ||= []
           event[:tags] << error_tag
         end
-        event
+        event[target] = Time.now.utc.iso8601(3)
+        filtered event, conf
       end
     end
   end

data/lib/anschel/output/device.rb

@@ -9,7 +9,7 @@ module Anschel
       def initialize config, stats, log
         log.trace event: 'output', kind: 'device', config: config
 
-        path   = config.delete(:path)
+        path   = config.delete(:path) || '/dev/stdout'
         qsize  = config.delete(:queue_size) || 2000
         @queue = SizedQueue.new qsize
 

data/lib/anschel/output/elasticsearch.rb

@@ -12,6 +12,7 @@ module Anschel
     class Elasticsearch < Base
       def initialize config, stats, log
         log.trace event: 'output', kind: 'elasticsearch', config: config
+        default_index = config.delete(:default_index) || 'logs-anschel'
         qsize   = config.delete(:queue_size) || 2000
         bsize   = config.delete(:bulk_size) || 500
         timeout = config.delete(:bulk_timeout) || 0.5
@@ -37,7 +38,15 @@ module Anschel
             next if events.empty?
 
             body = events.map do |e|
-              index = e.delete(:_index) || 'logs-anschel'
+              index = e.delete(:_index)
+              if index.nil?
+                log.error \
+                  event: 'elasticsearch-output-error',
+                  reason: 'event was not indexed',
+                  remediation: "sending to default index '#{default_index}'",
+                  raw_event: e
+                index = default_index
+              end
               { index: { _index: index, _type: e[:type], data: e } }
             end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-29 00:00:00.000000000 Z
+date: 2015-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement