anschel 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6a2475c658cf220a706e32d1bd87816b1278465d
4
- data.tar.gz: b6a9b99ff3ce1b8829373c1abc08809f9b0e7660
3
+ metadata.gz: 8f81ae5fdcf71f580680520067255911285629aa
4
+ data.tar.gz: 68ab5e077fa13d9d4e142fe798f222912dce5f6a
5
5
  SHA512:
6
- metadata.gz: 5e6882bb74eb0f96074c0a126c67eb9151d49ad64ab094fc91089cbdc9e3ae5d8603748b21e3190c6192defea4885d2517f632270dde4ac5a38ac3efc4a65e95
7
- data.tar.gz: fd482f04a614e36a9f75a104e77ed52f2905e80730a35aabe485d8c2cce20918283eec6a2e447c9c3ff9b0dc347c41cd19107ae1c1a8020f6d5f5e215ef18f24
6
+ metadata.gz: 0179da5dc7e53fc18e3c490f4e49f521e210184a0df80a5d834052237b2f476f4b99fbcad40bd390f1f318a9af9559f9609117098d8e7cf02d35829a560d2aab
7
+ data.tar.gz: 95309336805570a5e7cd50dfa1d47e43677caa3a72f78ea1a8e8b7fadc68146c26e00feb73706b0a2620868f73777af78b94c4b6368eeb8f4bc1c8e30c74d88f
data/Readme.md CHANGED
@@ -54,10 +54,13 @@ It's kinda like a JSON version of the Logstash config language:
54
54
  "pattern": "[%d] %p %m (%c)%n"
55
55
  },
56
56
 
57
- // Kafka is the primary input; see the `jruby-kafka` homepage for
58
- // more details: https://github.com/joekiller/jruby-kafka
59
- "input": {
60
- "kafka": {
57
+ // Specify any number of inputs
58
+ "input": [
59
+
60
+ // Kafka is the primary input; see the `jruby-kafka` homepage for
61
+ // more details: https://github.com/joekiller/jruby-kafka
62
+ {
63
+ "kind": "kafka",
61
64
  "queue_size": 2000,
62
65
  "zk_connect": "localhost:2181",
63
66
  "zk_connect_timeout": 6000,
@@ -81,12 +84,15 @@ It's kinda like a JSON version of the Logstash config language:
81
84
  "consumer_timeout_ms": -1,
82
85
  "consumer_restart_sleep_ms": 0
83
86
  }
84
- },
87
+ ],
88
+
89
+ // Specify any number of outputs
90
+ "output": [
85
91
 
86
- // Elasticsearch is the primary output; see the `elasticsearch-ruby`
87
- // homepage for more: https://github.com/elastic/elasticsearch-ruby
88
- "output": {
89
- "elasticsearch": {
92
+ // Elasticsearch is the primary output; see the `elasticsearch-ruby`
93
+ // homepage for more: https://github.com/elastic/elasticsearch-ruby
94
+ {
95
+ "kind": "elasticsearch",
90
96
  "queue_size": 2000,
91
97
  "bulk_size": 200,
92
98
  "hosts": [ "localhost:9200" ],
@@ -95,7 +101,7 @@ It's kinda like a JSON version of the Logstash config language:
95
101
  "reload_on_failure": true,
96
102
  "sniffer_timeout": 5
97
103
  }
98
- },
104
+ ],
99
105
 
100
106
  // Just like Logstash, Anschel has a notion of filters
101
107
  "filter": {
@@ -148,7 +154,6 @@ You might deploy Anschel with Upstart. Here's a minimal config:
148
154
 
149
155
  start on startup
150
156
  stop on shutdown
151
- respawn
152
157
 
153
158
  exec java -jar anschel-x.y.z.jar \
154
159
  --config /etc/anschel.json --log /var/log/anschel.log
@@ -161,7 +166,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
161
166
 
162
167
  _In development_
163
168
 
164
- - Intial implementation of the Kafka-to-Elasticsearch pipeline
165
- - Support for file (device) output (v0.4)
169
+ - Allow multiple input and output configurations (v0.6)
166
170
  - Support for RabbitMQ input (v0.5)
167
- - Allow multiple input and output configuration (v0.6)
171
+ - Support for file (device) output (v0.4)
172
+ - Intial implementation of the Kafka-to-Elasticsearch pipeline
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.0
1
+ 0.6.1
@@ -12,7 +12,9 @@ module Anschel
12
12
  stamp = conf.delete(:stamp) || '@timestamp'
13
13
  prefix = conf.delete(:prefix) || 'logs-%{type}-'
14
14
  suffix = conf.delete(:suffix) || '%Y.%m.%d'
15
- format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZ" # ISO8601
15
+ format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" # ISO8601
16
+
17
+ error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'index-error'
16
18
 
17
19
  stamp = stamp.to_sym
18
20
 
@@ -26,11 +28,26 @@ module Anschel
26
28
 
27
29
  lambda do |event|
28
30
  return event unless event.has_key? stamp
29
- millis = joda.parseMillis event[stamp]
30
31
  idx_prefix = prefix % event
31
- idx_suffix = Time.at(0.001 * millis).strftime(suffix)
32
- event[:_index] = idx_prefix + idx_suffix
33
- filtered event, conf
32
+ begin
33
+ millis = joda.parseMillis event[stamp]
34
+ idx_suffix = Time.at(0.001 * millis).strftime(suffix)
35
+ event[:_index] = idx_prefix + idx_suffix
36
+ filtered event, conf
37
+ rescue java.lang.IllegalArgumentException => e
38
+ event[:_index] = idx_prefix + Time.now.strftime(suffix)
39
+ log.warn \
40
+ event: 'filter-index-warning',
41
+ reason: 'could not parse event',
42
+ remediation: 'added bogus index',
43
+ remediation: "sending to best-guess index '#{event[:_index]}'",
44
+ raw_event: event
45
+ if error_tag
46
+ event[:tags] ||= []
47
+ event[:tags] << error_tag
48
+ end
49
+ filtered event, conf
50
+ end
34
51
  end
35
52
  end
36
53
  end
@@ -22,6 +22,15 @@ module Anschel
22
22
  lambda do |event|
23
23
  return event unless event.has_key? field
24
24
  mdata = pattern.match event[field]
25
+ if mdata.nil?
26
+ log.error \
27
+ event: 'parse-filter-error',
28
+ reason: 'regexp did not match',
29
+ field: field,
30
+ pattern: pattern,
31
+ raw_event: event
32
+ return filtered(event, conf)
33
+ end
25
34
  mdata.names.each do |group|
26
35
  event[group.to_sym] = mdata[group]
27
36
  end
@@ -1,5 +1,6 @@
1
1
  # {
2
2
  # "stamp": {
3
+ # "utc?": false,
3
4
  # "field": "timestamp",
4
5
  # "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
5
6
  # }
@@ -7,6 +8,7 @@
7
8
  module Anschel
8
9
  class Filter
9
10
  def stamp conf, log
11
+ utc = conf.delete :utc?
10
12
  field = conf.delete :field
11
13
  pattern = conf.delete :pattern
12
14
  target = conf.delete :target
@@ -27,6 +29,7 @@ module Anschel
27
29
  joda = joda.withOffsetParsed
28
30
  end
29
31
 
32
+ offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
30
33
 
31
34
  log.debug event: 'compiled-filter', filter: 'stamp', \
32
35
  field: field, pattern: pattern, target: target, error_tag: error_tag
@@ -34,19 +37,25 @@ module Anschel
34
37
  lambda do |event|
35
38
  return event unless event.has_key? field
36
39
  parsers.each do |joda|
37
- millis = joda.parseMillis event[field]
38
40
  begin
39
- event[target] = Time.at(0.001 * millis).iso8601(3)
41
+ millis = joda.parseMillis event[field]
42
+ event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
40
43
  return filtered(event, conf)
41
44
  rescue
42
45
  end
43
46
  end
44
47
 
48
+ log.warn \
49
+ event: 'stamp-filter-warning',
50
+ reason: 'could not parse event',
51
+ remediation: 'using current time for stamp',
52
+ raw_event: event
45
53
  if error_tag
46
54
  event[:tags] ||= []
47
55
  event[:tags] << error_tag
48
56
  end
49
- event
57
+ event[target] = Time.now.utc.iso8601(3)
58
+ filtered event, conf
50
59
  end
51
60
  end
52
61
  end
@@ -9,7 +9,7 @@ module Anschel
9
9
  def initialize config, stats, log
10
10
  log.trace event: 'output', kind: 'device', config: config
11
11
 
12
- path = config.delete(:path)
12
+ path = config.delete(:path) || '/dev/stdout'
13
13
  qsize = config.delete(:queue_size) || 2000
14
14
  @queue = SizedQueue.new qsize
15
15
 
@@ -12,6 +12,7 @@ module Anschel
12
12
  class Elasticsearch < Base
13
13
  def initialize config, stats, log
14
14
  log.trace event: 'output', kind: 'elasticsearch', config: config
15
+ default_index = config.delete(:default_index) || 'logs-anschel'
15
16
  qsize = config.delete(:queue_size) || 2000
16
17
  bsize = config.delete(:bulk_size) || 500
17
18
  timeout = config.delete(:bulk_timeout) || 0.5
@@ -37,7 +38,15 @@ module Anschel
37
38
  next if events.empty?
38
39
 
39
40
  body = events.map do |e|
40
- index = e.delete(:_index) || 'logs-anschel'
41
+ index = e.delete(:_index)
42
+ if index.nil?
43
+ log.error \
44
+ event: 'elasticsearch-output-error',
45
+ reason: 'event was not indexed',
46
+ remediation: "sending to default index '#{default_index}'",
47
+ raw_event: e
48
+ index = default_index
49
+ end
41
50
  { index: { _index: index, _type: e[:type], data: e } }
42
51
  end
43
52
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: anschel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Clemmer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-29 00:00:00.000000000 Z
11
+ date: 2015-09-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement