anschel 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16a3a2475aa25d79231c2ad9ff8efe4730238268
-  data.tar.gz: e253c129900cd38fcc9ee6fe64818997ba3b22c3
+  metadata.gz: 1c067fd0f07d2642121198aed3e1a129c753ed34
+  data.tar.gz: 4c404083a4512a68bbe8f84683e6d89dd47e3acd
 SHA512:
-  metadata.gz: 6415a8a2ee00b8cbdc0a57174213b7ec48df7cd4312a56c581a97b2de2fa108e66b217dc0806ff6d81acbf6c59ed4327de8389376c28553ea821771b7e910fe9
-  data.tar.gz: 3b9ab4e50c6ac74b264248506e8ad65f912cef6e3d8a59295ca5567d6d7f12f2734d1e716e1294ac84ca0e77a6ec0e3a1f2181c1b261384c70e5fd4b7fa2bca2
+  metadata.gz: 62fd8c2b1775ae0d938473fd706d2520d5f6dfa6573184f540e0fff7f3d91dacd93e18ca4fdfc2da06d89b13eba830248d163293b7c0aed744055e898014ebf6
+  data.tar.gz: 2f986177c04b3a24af8c48966f270b8bed5d94b26db59147f866b98f91b493c4b552de21a23e54e11d8c842ede68a345ac06e6c07ab374fd32039049dc70a9e5

data/Readme.md

@@ -54,43 +54,47 @@ It's kinda like a JSON version of the Logstash config language:
         "pattern": "[%d] %p %m (%c)%n"
       },
 
-      // Kafka is the only supported input; see the `jruby-kafka` homepage for
+      // Kafka is the primary input; see the `jruby-kafka` homepage for
       // more details: https://github.com/joekiller/jruby-kafka
-      "kafka": {
-        "queue_size": 2000,
-        "zk_connect": "localhost:2181",
-        "zk_connect_timeout": 6000,
-        "zk_session_timeout": 6000,
-        "group_id": "anschel",
-        "topic_id": "franz",
-        "reset_beginning": null,
-        "auto_offset_reset": "smallest",
-        "consumer_restart_on_error": true,
-        "auto_commit_interval": 1000,
-        "rebalance_max_retries": 4,
-        "rebalance_backoff_ms": 2000,
-        "socket_timeout_ms": 30000,
-        "socket_receive_buffer_bytes": 65536,
-        "fetch_message_max_bytes": 1048576,
-        "auto_commit_enable": true,
-        "queued_max_message_chunks": 10,
-        "fetch_min_bytes": 1,
-        "fetch_wait_max_ms": 100,
-        "refresh_leader_backoff_ms": 200,
-        "consumer_timeout_ms": -1,
-        "consumer_restart_sleep_ms": 0
+      "input": {
+        "kafka": {
+          "queue_size": 2000,
+          "zk_connect": "localhost:2181",
+          "zk_connect_timeout": 6000,
+          "zk_session_timeout": 6000,
+          "group_id": "anschel",
+          "topic_id": "franz",
+          "reset_beginning": null,
+          "auto_offset_reset": "smallest",
+          "consumer_restart_on_error": true,
+          "auto_commit_interval": 1000,
+          "rebalance_max_retries": 4,
+          "rebalance_backoff_ms": 2000,
+          "socket_timeout_ms": 30000,
+          "socket_receive_buffer_bytes": 65536,
+          "fetch_message_max_bytes": 1048576,
+          "auto_commit_enable": true,
+          "queued_max_message_chunks": 10,
+          "fetch_min_bytes": 1,
+          "fetch_wait_max_ms": 100,
+          "refresh_leader_backoff_ms": 200,
+          "consumer_timeout_ms": -1,
+          "consumer_restart_sleep_ms": 0
+        }
       },
 
-      // There's only one output, Elasticsearch; see the `elasticsearch-ruby`
+      // Elasticsearch is the primary output; see the `elasticsearch-ruby`
       // homepage for more: https://github.com/elastic/elasticsearch-ruby
-      "elasticsearch": {
-        "queue_size": 2000,
-        "bulk_size": 200,
-        "hosts": [ "localhost:9200" ],
-        "randomize_hosts": true,
-        "reload_connections": true,
-        "reload_on_failure": true,
-        "sniffer_timeout": 5
+      "output": {
+        "elasticsearch": {
+          "queue_size": 2000,
+          "bulk_size": 200,
+          "hosts": [ "localhost:9200" ],
+          "randomize_hosts": true,
+          "reload_connections": true,
+          "reload_on_failure": true,
+          "sniffer_timeout": 5
+        }
       },
 
       // Just like Logstash, Anschel has a notion of filters
@@ -155,6 +159,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
 
 ### v1.0
 
-_In devlopment_
+_In development_
 
 - Intial implementation of the Kafka-to-Elasticsearch pipeline
+- Additional support for RabbitMQ input and file (device) output

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.5.0

data/lib/anschel/input/kafka.rb

@@ -0,0 +1,27 @@
+require 'jruby-kafka'
+
+
+module Anschel
+  class Input
+    class Kafka
+      def initialize config, stats, log
+        log.trace event: 'input', kind: 'kafka', config: config
+        qsize  = config.delete(:queue_size) || 1000
+        @queue = SizedQueue.new qsize
+        consumer_group = ::Kafka::Group.new config
+        consumer_group.run num_cpus, @queue
+
+        trap('SIGINT') do
+          consumer_group.shutdown
+          log.info event: 'goodbye', version: VERSION
+          exit
+        end
+
+        log.info event: 'input-loaded'
+      end
+
+
+      def shift ; @queue.shift end
+    end
+  end
+end

data/lib/anschel/input/rabbitmq.rb

@@ -0,0 +1,52 @@
+require 'march_hare'
+
+
+module Anschel
+  class Input
+    class RabbitMQ
+      def initialize config, stats, log
+        log.trace event: 'input', kind: 'rabbitmq', config: config
+        qsize  = config[:connection].delete(:queue_size) || 1000
+        @queue = SizedQueue.new qsize
+
+        default_exchange = { type: 'x-consistent-hash' }
+
+        default_qconf = {
+          exclusive: false,
+          auto_delete: false,
+          durable: true
+        }
+
+        exchange_name = config[:exchange].delete(:name)
+
+        rmq_threads = config[:queues].map do |qname, qconf|
+          Thread.new do
+            conn = ::MarchHare.connect config[:connection]
+            chan = conn.create_channel
+
+            exchange = channel.exchange exchange_name, \
+              default_exchange.merge(config[:exchange])
+
+            rmq = chan.queue qname, default_qconf.merge(qconf)
+            rmq.subscribe(block: true, ack: true) do |_, message|
+              @queue << message
+            end
+          end
+        end
+
+        trap('SIGINT') do
+          rmq_threads.map &:kill
+          log.info event: 'goodbye', version: VERSION
+          exit
+        end
+
+        log.info event: 'input-loaded'
+
+        rmq_threads.map &:join
+      end
+
+
+      def shift ; @queue.shift end
+    end
+  end
+end

data/lib/anschel/input.rb

@@ -1,25 +1,17 @@
-require 'jruby-kafka'
+require_relative 'input/kafka'
+require_relative 'input/rabbitmq'
 
 
 module Anschel
   class Input
     def initialize config, stats, log
-      log.trace event: 'input', config: config
-      qsize  = config.delete(:queue_size) || 1000
-      @queue = SizedQueue.new qsize
-      consumer_group = Kafka::Group.new config
-      consumer_group.run num_cpus, @queue
-
-      trap('SIGINT') do
-        consumer_group.shutdown
-        log.info event: 'goodbye', version: VERSION
-        exit
+      if config[:kafka]
+        Input::Kafka.new config[:kafka], stats, log
+      elsif config[:rabbitmq]
+        Input::RabbitMQ.new config[:rabbitmq], stats, log
+      else
+        raise 'Unkown input type'
       end
-
-      log.info event: 'input-loaded'
     end
-
-
-    def shift ; @queue.shift end
   end
 end

data/lib/anschel/main.rb

@@ -50,14 +50,12 @@ module Anschel
       setup_log4j config[:log4j]
 
       stats  = Stats.new log, options.stats_interval
-      input  = Input.new config[:kafka], stats, log
+
+      input = Input.new config[:input], stats, log
+
       filter = Filter.new config[:filter], stats, log
 
-      output = if config[:device]
-        Output::Device.new config[:device], stats, log
-      else
-        Output::Elasticsearch.new config[:elasticsearch], stats, log
-      end
+      output = Output.new config[:output], stats, log
 
       stats.create 'event'
       stats.get 'event'

data/lib/anschel/output/device.rb

@@ -1,7 +1,7 @@
 require 'thread'
 
 module Anschel
-  module Output
+  class Output
     class Device
       def initialize config, stats, log
         log.trace event: 'output', kind: 'device', config: config

data/lib/anschel/output/elasticsearch.rb

@@ -6,7 +6,7 @@ require 'elasticsearch'
 
 
 module Anschel
-  module Output
+  class Output
     class Elasticsearch
       def initialize config, stats, log
         log.trace event: 'output', kind: 'elasticsearch', config: config

data/lib/anschel/output.rb

@@ -1,2 +1,17 @@
 require_relative 'output/elasticsearch'
-require_relative 'output/device'
+require_relative 'output/device'
+
+
+module Anschel
+  class Output
+    def initialize config, stats, log
+      if config[:device]
+        Output::Device.new config[:device], stats, log
+      elsif config[:elasticsearch]
+        Output::Elasticsearch.new config[:elasticsearch], stats, log
+      else
+        raise 'Unkown output type'
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  name: march_hare
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
 description: Logstash-like for moving events from Kafka into Elasticsearch.
 email: sczizzo@gmail.com
 executables:
@@ -114,6 +128,8 @@ files:
 - lib/anschel/filter/scan.rb
 - lib/anschel/filter/stamp.rb
 - lib/anschel/input.rb
+- lib/anschel/input/kafka.rb
+- lib/anschel/input/rabbitmq.rb
 - lib/anschel/main.rb
 - lib/anschel/metadata.rb
 - lib/anschel/mjolnir.rb