anschel 0.4.1 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 16a3a2475aa25d79231c2ad9ff8efe4730238268
4
- data.tar.gz: e253c129900cd38fcc9ee6fe64818997ba3b22c3
3
+ metadata.gz: 1c067fd0f07d2642121198aed3e1a129c753ed34
4
+ data.tar.gz: 4c404083a4512a68bbe8f84683e6d89dd47e3acd
5
5
  SHA512:
6
- metadata.gz: 6415a8a2ee00b8cbdc0a57174213b7ec48df7cd4312a56c581a97b2de2fa108e66b217dc0806ff6d81acbf6c59ed4327de8389376c28553ea821771b7e910fe9
7
- data.tar.gz: 3b9ab4e50c6ac74b264248506e8ad65f912cef6e3d8a59295ca5567d6d7f12f2734d1e716e1294ac84ca0e77a6ec0e3a1f2181c1b261384c70e5fd4b7fa2bca2
6
+ metadata.gz: 62fd8c2b1775ae0d938473fd706d2520d5f6dfa6573184f540e0fff7f3d91dacd93e18ca4fdfc2da06d89b13eba830248d163293b7c0aed744055e898014ebf6
7
+ data.tar.gz: 2f986177c04b3a24af8c48966f270b8bed5d94b26db59147f866b98f91b493c4b552de21a23e54e11d8c842ede68a345ac06e6c07ab374fd32039049dc70a9e5
data/Readme.md CHANGED
@@ -54,43 +54,47 @@ It's kinda like a JSON version of the Logstash config language:
54
54
  "pattern": "[%d] %p %m (%c)%n"
55
55
  },
56
56
 
57
- // Kafka is the only supported input; see the `jruby-kafka` homepage for
57
+ // Kafka is the primary input; see the `jruby-kafka` homepage for
58
58
  // more details: https://github.com/joekiller/jruby-kafka
59
- "kafka": {
60
- "queue_size": 2000,
61
- "zk_connect": "localhost:2181",
62
- "zk_connect_timeout": 6000,
63
- "zk_session_timeout": 6000,
64
- "group_id": "anschel",
65
- "topic_id": "franz",
66
- "reset_beginning": null,
67
- "auto_offset_reset": "smallest",
68
- "consumer_restart_on_error": true,
69
- "auto_commit_interval": 1000,
70
- "rebalance_max_retries": 4,
71
- "rebalance_backoff_ms": 2000,
72
- "socket_timeout_ms": 30000,
73
- "socket_receive_buffer_bytes": 65536,
74
- "fetch_message_max_bytes": 1048576,
75
- "auto_commit_enable": true,
76
- "queued_max_message_chunks": 10,
77
- "fetch_min_bytes": 1,
78
- "fetch_wait_max_ms": 100,
79
- "refresh_leader_backoff_ms": 200,
80
- "consumer_timeout_ms": -1,
81
- "consumer_restart_sleep_ms": 0
59
+ "input": {
60
+ "kafka": {
61
+ "queue_size": 2000,
62
+ "zk_connect": "localhost:2181",
63
+ "zk_connect_timeout": 6000,
64
+ "zk_session_timeout": 6000,
65
+ "group_id": "anschel",
66
+ "topic_id": "franz",
67
+ "reset_beginning": null,
68
+ "auto_offset_reset": "smallest",
69
+ "consumer_restart_on_error": true,
70
+ "auto_commit_interval": 1000,
71
+ "rebalance_max_retries": 4,
72
+ "rebalance_backoff_ms": 2000,
73
+ "socket_timeout_ms": 30000,
74
+ "socket_receive_buffer_bytes": 65536,
75
+ "fetch_message_max_bytes": 1048576,
76
+ "auto_commit_enable": true,
77
+ "queued_max_message_chunks": 10,
78
+ "fetch_min_bytes": 1,
79
+ "fetch_wait_max_ms": 100,
80
+ "refresh_leader_backoff_ms": 200,
81
+ "consumer_timeout_ms": -1,
82
+ "consumer_restart_sleep_ms": 0
83
+ }
82
84
  },
83
85
 
84
- // There's only one output, Elasticsearch; see the `elasticsearch-ruby`
86
+ // Elasticsearch is the primary output; see the `elasticsearch-ruby`
85
87
  // homepage for more: https://github.com/elastic/elasticsearch-ruby
86
- "elasticsearch": {
87
- "queue_size": 2000,
88
- "bulk_size": 200,
89
- "hosts": [ "localhost:9200" ],
90
- "randomize_hosts": true,
91
- "reload_connections": true,
92
- "reload_on_failure": true,
93
- "sniffer_timeout": 5
88
+ "output": {
89
+ "elasticsearch": {
90
+ "queue_size": 2000,
91
+ "bulk_size": 200,
92
+ "hosts": [ "localhost:9200" ],
93
+ "randomize_hosts": true,
94
+ "reload_connections": true,
95
+ "reload_on_failure": true,
96
+ "sniffer_timeout": 5
97
+ }
94
98
  },
95
99
 
96
100
  // Just like Logstash, Anschel has a notion of filters
@@ -155,6 +159,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
155
159
 
156
160
  ### v1.0
157
161
 
158
- _In devlopment_
162
+ _In development_
159
163
 
160
164
  - Intial implementation of the Kafka-to-Elasticsearch pipeline
165
+ - Additional support for RabbitMQ input and file (device) output
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.1
1
+ 0.5.0
@@ -0,0 +1,27 @@
1
+ require 'jruby-kafka'
2
+
3
+
4
+ module Anschel
5
+ class Input
6
+ class Kafka
7
+ def initialize config, stats, log
8
+ log.trace event: 'input', kind: 'kafka', config: config
9
+ qsize = config.delete(:queue_size) || 1000
10
+ @queue = SizedQueue.new qsize
11
+ consumer_group = ::Kafka::Group.new config
12
+ consumer_group.run num_cpus, @queue
13
+
14
+ trap('SIGINT') do
15
+ consumer_group.shutdown
16
+ log.info event: 'goodbye', version: VERSION
17
+ exit
18
+ end
19
+
20
+ log.info event: 'input-loaded'
21
+ end
22
+
23
+
24
+ def shift ; @queue.shift end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,52 @@
1
+ require 'march_hare'
2
+
3
+
4
+ module Anschel
5
+ class Input
6
+ class RabbitMQ
7
+ def initialize config, stats, log
8
+ log.trace event: 'input', kind: 'rabbitmq', config: config
9
+ qsize = config[:connection].delete(:queue_size) || 1000
10
+ @queue = SizedQueue.new qsize
11
+
12
+ default_exchange = { type: 'x-consistent-hash' }
13
+
14
+ default_qconf = {
15
+ exclusive: false,
16
+ auto_delete: false,
17
+ durable: true
18
+ }
19
+
20
+ exchange_name = config[:exchange].delete(:name)
21
+
22
+ rmq_threads = config[:queues].map do |qname, qconf|
23
+ Thread.new do
24
+ conn = ::MarchHare.connect config[:connection]
25
+ chan = conn.create_channel
26
+
27
+ exchange = channel.exchange exchange_name, \
28
+ default_exchange.merge(config[:exchange])
29
+
30
+ rmq = chan.queue qname, default_qconf.merge(qconf)
31
+ rmq.subscribe(block: true, ack: true) do |_, message|
32
+ @queue << message
33
+ end
34
+ end
35
+ end
36
+
37
+ trap('SIGINT') do
38
+ rmq_threads.map &:kill
39
+ log.info event: 'goodbye', version: VERSION
40
+ exit
41
+ end
42
+
43
+ log.info event: 'input-loaded'
44
+
45
+ rmq_threads.map &:join
46
+ end
47
+
48
+
49
+ def shift ; @queue.shift end
50
+ end
51
+ end
52
+ end
data/lib/anschel/input.rb CHANGED
@@ -1,25 +1,17 @@
1
- require 'jruby-kafka'
1
+ require_relative 'input/kafka'
2
+ require_relative 'input/rabbitmq'
2
3
 
3
4
 
4
5
  module Anschel
5
6
  class Input
6
7
  def initialize config, stats, log
7
- log.trace event: 'input', config: config
8
- qsize = config.delete(:queue_size) || 1000
9
- @queue = SizedQueue.new qsize
10
- consumer_group = Kafka::Group.new config
11
- consumer_group.run num_cpus, @queue
12
-
13
- trap('SIGINT') do
14
- consumer_group.shutdown
15
- log.info event: 'goodbye', version: VERSION
16
- exit
8
+ if config[:kafka]
9
+ Input::Kafka.new config[:kafka], stats, log
10
+ elsif config[:rabbitmq]
11
+ Input::RabbitMQ.new config[:rabbitmq], stats, log
12
+ else
13
+ raise 'Unkown input type'
17
14
  end
18
-
19
- log.info event: 'input-loaded'
20
15
  end
21
-
22
-
23
- def shift ; @queue.shift end
24
16
  end
25
17
  end
data/lib/anschel/main.rb CHANGED
@@ -50,14 +50,12 @@ module Anschel
50
50
  setup_log4j config[:log4j]
51
51
 
52
52
  stats = Stats.new log, options.stats_interval
53
- input = Input.new config[:kafka], stats, log
53
+
54
+ input = Input.new config[:input], stats, log
55
+
54
56
  filter = Filter.new config[:filter], stats, log
55
57
 
56
- output = if config[:device]
57
- Output::Device.new config[:device], stats, log
58
- else
59
- Output::Elasticsearch.new config[:elasticsearch], stats, log
60
- end
58
+ output = Output.new config[:output], stats, log
61
59
 
62
60
  stats.create 'event'
63
61
  stats.get 'event'
@@ -1,7 +1,7 @@
1
1
  require 'thread'
2
2
 
3
3
  module Anschel
4
- module Output
4
+ class Output
5
5
  class Device
6
6
  def initialize config, stats, log
7
7
  log.trace event: 'output', kind: 'device', config: config
@@ -6,7 +6,7 @@ require 'elasticsearch'
6
6
 
7
7
 
8
8
  module Anschel
9
- module Output
9
+ class Output
10
10
  class Elasticsearch
11
11
  def initialize config, stats, log
12
12
  log.trace event: 'output', kind: 'elasticsearch', config: config
@@ -1,2 +1,17 @@
1
1
  require_relative 'output/elasticsearch'
2
- require_relative 'output/device'
2
+ require_relative 'output/device'
3
+
4
+
5
+ module Anschel
6
+ class Output
7
+ def initialize config, stats, log
8
+ if config[:device]
9
+ Output::Device.new config[:device], stats, log
10
+ elsif config[:elasticsearch]
11
+ Output::Elasticsearch.new config[:elasticsearch], stats, log
12
+ else
13
+ raise 'Unkown output type'
14
+ end
15
+ end
16
+ end
17
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: anschel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Clemmer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-30 00:00:00.000000000 Z
11
+ date: 2015-09-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ~>
95
95
  - !ruby/object:Gem::Version
96
96
  version: '1.1'
97
+ - !ruby/object:Gem::Dependency
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ~>
101
+ - !ruby/object:Gem::Version
102
+ version: '2.12'
103
+ name: march_hare
104
+ prerelease: false
105
+ type: :runtime
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ~>
109
+ - !ruby/object:Gem::Version
110
+ version: '2.12'
97
111
  description: Logstash-like for moving events from Kafka into Elasticsearch.
98
112
  email: sczizzo@gmail.com
99
113
  executables:
@@ -114,6 +128,8 @@ files:
114
128
  - lib/anschel/filter/scan.rb
115
129
  - lib/anschel/filter/stamp.rb
116
130
  - lib/anschel/input.rb
131
+ - lib/anschel/input/kafka.rb
132
+ - lib/anschel/input/rabbitmq.rb
117
133
  - lib/anschel/main.rb
118
134
  - lib/anschel/metadata.rb
119
135
  - lib/anschel/mjolnir.rb