anschel 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c067fd0f07d2642121198aed3e1a129c753ed34
-  data.tar.gz: 4c404083a4512a68bbe8f84683e6d89dd47e3acd
+  metadata.gz: 6a2475c658cf220a706e32d1bd87816b1278465d
+  data.tar.gz: b6a9b99ff3ce1b8829373c1abc08809f9b0e7660
 SHA512:
-  metadata.gz: 62fd8c2b1775ae0d938473fd706d2520d5f6dfa6573184f540e0fff7f3d91dacd93e18ca4fdfc2da06d89b13eba830248d163293b7c0aed744055e898014ebf6
-  data.tar.gz: 2f986177c04b3a24af8c48966f270b8bed5d94b26db59147f866b98f91b493c4b552de21a23e54e11d8c842ede68a345ac06e6c07ab374fd32039049dc70a9e5
+  metadata.gz: 5e6882bb74eb0f96074c0a126c67eb9151d49ad64ab094fc91089cbdc9e3ae5d8603748b21e3190c6192defea4885d2517f632270dde4ac5a38ac3efc4a65e95
+  data.tar.gz: fd482f04a614e36a9f75a104e77ed52f2905e80730a35aabe485d8c2cce20918283eec6a2e447c9c3ff9b0dc347c41cd19107ae1c1a8020f6d5f5e215ef18f24

data/Readme.md

@@ -162,4 +162,6 @@ You might deploy Anschel with Upstart. Here's a minimal config:
 _In development_
 
 - Intial implementation of the Kafka-to-Elasticsearch pipeline
-- Additional support for RabbitMQ input and file (device) output
+- Support for file (device) output (v0.4)
+- Support for RabbitMQ input (v0.5)
+- Allow multiple input and output configuration (v0.6)

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/lib/anschel/filter/index.rb

@@ -2,7 +2,8 @@
 #   "index": {
 #     "stamp": "",
 #     "prefix": "",
-#     "suffix": ""
+#     "suffix": "",
+#     "format": ""
 #   }
 # }
 module Anschel
@@ -11,18 +12,17 @@ module Anschel
       stamp  = conf.delete(:stamp)  || '@timestamp'
       prefix = conf.delete(:prefix) || 'logs-%{type}-'
       suffix = conf.delete(:suffix) || '%Y.%m.%d'
+      format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZ" # ISO8601
 
       stamp = stamp.to_sym
 
-      iso8601 = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
-
-      joda = org.joda.time.format.DateTimeFormat.forPattern iso8601
+      joda = org.joda.time.format.DateTimeFormat.forPattern format
       joda = joda.withDefaultYear(Time.new.year)
       joda = joda.withOffsetParsed
 
 
       log.debug event: 'compiled-filter', filter: 'index', \
-        stamp: stamp, prefix: prefix, suffix: suffix
+        stamp: stamp, prefix: prefix, suffix: suffix, format: format
 
       lambda do |event|
         return event unless event.has_key? stamp

data/lib/anschel/filter/parse.rb

@@ -23,7 +23,7 @@ module Anschel
         return event unless event.has_key? field
         mdata = pattern.match event[field]
         mdata.names.each do |group|
-          event[group] = mdata[group]
+          event[group.to_sym] = mdata[group]
         end
         filtered event, conf
       end

data/lib/anschel/filter.rb

@@ -28,7 +28,7 @@ module Anschel
 
     def apply event
       raise 'Event does not have a "type" field' unless event[:type]
-      type = event[:type].dup # In case of modification
+      type = event[:type].to_sym # In case of modification
       filters[:_before].each { |f| f.call event }
       filters[type].each     { |f| f.call event }
       filters[:_after].each  { |f| f.call event }
@@ -40,7 +40,7 @@ module Anschel
 
     def filtered event, options
       if remove_field = options[:remove_field]
-        event.delete remove_field
+        event.delete remove_field.to_sym
       end
       event
     end

data/lib/anschel/input/base.rb

@@ -0,0 +1,9 @@
+module Anschel
+  class Input
+    class Base
+      def stop
+        raise 'To be implemented'
+      end
+    end
+  end
+end

data/lib/anschel/input/kafka.rb

@@ -1,27 +1,22 @@
 require 'jruby-kafka'
 
+require_relative 'base'
+
 
 module Anschel
   class Input
-    class Kafka
-      def initialize config, stats, log
+    class Kafka < Base
+      def initialize queue, config, stats, log
         log.trace event: 'input', kind: 'kafka', config: config
-        qsize  = config.delete(:queue_size) || 1000
-        @queue = SizedQueue.new qsize
-        consumer_group = ::Kafka::Group.new config
-        consumer_group.run num_cpus, @queue
-
-        trap('SIGINT') do
-          consumer_group.shutdown
-          log.info event: 'goodbye', version: VERSION
-          exit
-        end
-
-        log.info event: 'input-loaded'
+        @consumer_group = ::Kafka::Group.new config
+        @consumer_group.run num_cpus, queue
       end
 
-
-      def shift ; @queue.shift end
+      def stop
+        return if @stopped
+        @consumer_group.shutdown
+        @stopped = true
+      end
     end
   end
 end

data/lib/anschel/input/rabbitmq.rb

@@ -1,15 +1,18 @@
 require 'march_hare'
 
+require_relative 'base'
+
 
 module Anschel
   class Input
-    class RabbitMQ
-      def initialize config, stats, log
+    class RabbitMQ < Base
+      def initialize queue, config, stats, log
         log.trace event: 'input', kind: 'rabbitmq', config: config
-        qsize  = config[:connection].delete(:queue_size) || 1000
-        @queue = SizedQueue.new qsize
 
-        default_exchange = { type: 'x-consistent-hash' }
+        default_exchange = {
+          type: 'x-consistent-hash',
+          durable: true
+        }
 
         default_qconf = {
           exclusive: false,
@@ -19,34 +22,27 @@ module Anschel
 
         exchange_name = config[:exchange].delete(:name)
 
-        rmq_threads = config[:queues].map do |qname, qconf|
+        @threads = config[:queues].map do |qname, qconf|
           Thread.new do
             conn = ::MarchHare.connect config[:connection]
             chan = conn.create_channel
 
-            exchange = channel.exchange exchange_name, \
+            exchange = chan.exchange exchange_name, \
               default_exchange.merge(config[:exchange])
 
-            rmq = chan.queue qname, default_qconf.merge(qconf)
-            rmq.subscribe(block: true, ack: true) do |_, message|
-              @queue << message
+            rmq = chan.queue qname.to_s, default_qconf.merge(qconf)
+            rmq.subscribe(block: true) do |_, message|
+              queue << message
             end
           end
         end
-
-        trap('SIGINT') do
-          rmq_threads.map &:kill
-          log.info event: 'goodbye', version: VERSION
-          exit
-        end
-
-        log.info event: 'input-loaded'
-
-        rmq_threads.map &:join
       end
 
-
-      def shift ; @queue.shift end
+      def stop
+        return if @stopped
+        @threads.map &:kill
+        @stopped = true
+      end
     end
   end
 end

data/lib/anschel/input.rb

@@ -4,13 +4,44 @@ require_relative 'input/rabbitmq'
 
 module Anschel
   class Input
-    def initialize config, stats, log
-      if config[:kafka]
-        Input::Kafka.new config[:kafka], stats, log
-      elsif config[:rabbitmq]
-        Input::RabbitMQ.new config[:rabbitmq], stats, log
+    def initialize config, qsize, stats, log
+      if config.empty?
+        raise 'No input provided'
+      end
+
+      @queue = SizedQueue.new qsize || 2000
+
+      @inputs = []
+
+      config.each do |input|
+        case input.delete(:kind)
+        when 'kafka'
+          @inputs << Input::Kafka.new(@queue, input, stats, log)
+        when 'rabbitmq'
+          @inputs << Input::RabbitMQ.new(@queue, input, stats, log)
+        else
+          raise 'Uknown input type'
+        end
+      end
+
+      log.info event: 'input-loaded'
+    end
+
+
+    def stop
+      return if @stopped
+      @inputs.map &:stop
+      @stopped = true
+    end
+
+
+    def shift
+      event = @queue.shift
+      case event
+      when String
+        event
       else
-        raise 'Unkown input type'
+        event.message.to_s
       end
     end
   end

data/lib/anschel/main.rb

@@ -46,17 +46,27 @@ module Anschel
         options: options.to_hash,
         num_cpus: num_cpus
 
-      config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
+      config = JrJackson::Json.load \
+        File.read(options.config), symbolize_keys: true
+
       setup_log4j config[:log4j]
 
       stats  = Stats.new log, options.stats_interval
 
-      input = Input.new config[:input], stats, log
+      input  = Input.new config[:input], config[:queue_size], stats, log
 
       filter = Filter.new config[:filter], stats, log
 
       output = Output.new config[:output], stats, log
 
+      trap('SIGINT') do
+        input.stop
+        output.stop
+        log.info event: 'goodbye', version: VERSION
+        exit
+      end
+
+
       stats.create 'event'
       stats.get 'event'
 
@@ -64,7 +74,7 @@ module Anschel
         Thread.new do
           loop do
             event = JrJackson::Json.load \
-              input.shift.message.to_s, symbolize_keys: true
+              input.shift, symbolize_keys: true
             output.push filter.apply(event)
             stats.inc 'event'
           end

data/lib/anschel/output/base.rb

@@ -0,0 +1,13 @@
+module Anschel
+  class Output
+    class Base
+      def stop
+        return if @stopped
+        @thread.kill
+        @stopped = true
+      end
+
+      def push event ; @queue.push event end
+    end
+  end
+end

data/lib/anschel/output/device.rb

@@ -1,27 +1,26 @@
 require 'thread'
 
+require_relative 'base'
+
+
 module Anschel
   class Output
-    class Device
+    class Device < Base
       def initialize config, stats, log
         log.trace event: 'output', kind: 'device', config: config
 
-        path  = config.delete(:path)
-        qsize = config.delete(:queue_size) || 2000
-        @q    = SizedQueue.new qsize
+        path   = config.delete(:path)
+        qsize  = config.delete(:queue_size) || 2000
+        @queue = SizedQueue.new qsize
 
-        Thread.new do
+        @thread = Thread.new do
           File.open(path, 'w') do |f|
             loop do
-              f.puts @q.shift
+              f.puts @queue.shift
             end
           end
         end
-
-        log.info event: 'output-loaded'
       end
-
-      def push event ; @q.push event end
     end
   end
 end

data/lib/anschel/output/elasticsearch.rb

@@ -1,16 +1,17 @@
 require 'thread'
 
-require 'typhoeus/adapters/faraday'
 require 'typhoeus'
+require 'typhoeus/adapters/faraday'
 require 'elasticsearch'
 
+require_relative 'base'
+
 
 module Anschel
   class Output
-    class Elasticsearch
+    class Elasticsearch < Base
       def initialize config, stats, log
         log.trace event: 'output', kind: 'elasticsearch', config: config
-        pattern = config.delete(:index_pattern)
         qsize   = config.delete(:queue_size) || 2000
         bsize   = config.delete(:bulk_size) || 500
         timeout = config.delete(:bulk_timeout) || 0.5
@@ -20,7 +21,7 @@ module Anschel
 
         @queue = SizedQueue.new qsize
 
-        Thread.new do
+        @thread = Thread.new do
           loop do
             events = []
             count  = 0
@@ -43,11 +44,7 @@ module Anschel
             client.bulk body: body
           end
         end
-
-        log.info event: 'output-loaded'
       end
-
-      def push event ; @queue.push event end
     end
   end
 end

data/lib/anschel/output.rb

@@ -5,12 +5,36 @@ require_relative 'output/device'
 module Anschel
   class Output
     def initialize config, stats, log
-      if config[:device]
-        Output::Device.new config[:device], stats, log
-      elsif config[:elasticsearch]
-        Output::Elasticsearch.new config[:elasticsearch], stats, log
-      else
-        raise 'Unkown output type'
+      if config.empty?
+        raise 'No output provided'
+      end
+
+      @outputs = []
+
+      config.each do |output|
+        case output.delete(:kind)
+        when 'device'
+          @outputs << Output::Device.new(output, stats, log)
+        when 'elasticsearch'
+          @outputs << Output::Elasticsearch.new(output, stats, log)
+        else
+          raise 'Unkown output type'
+        end
+      end
+
+      log.info event: 'output-loaded'
+    end
+
+
+    def stop
+      return if @stopped
+      @outputs.map &:stop
+      @stopped = true
+    end
+
+    def push event
+      @outputs.each do |output|
+        output.push event
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Sean Clemmer
@@ -128,12 +128,14 @@ files:
 - lib/anschel/filter/scan.rb
 - lib/anschel/filter/stamp.rb
 - lib/anschel/input.rb
+- lib/anschel/input/base.rb
 - lib/anschel/input/kafka.rb
 - lib/anschel/input/rabbitmq.rb
 - lib/anschel/main.rb
 - lib/anschel/metadata.rb
 - lib/anschel/mjolnir.rb
 - lib/anschel/output.rb
+- lib/anschel/output/base.rb
 - lib/anschel/output/device.rb
 - lib/anschel/output/elasticsearch.rb
 - lib/anschel/stats.rb