anschel 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4e693593d8211b7f7a7ee60d06536d463f74cb2f
4
- data.tar.gz: f67b0087ae0d56c73beed719263feb46e78ec301
3
+ metadata.gz: b20f566992eecc30983efa61d5a3154da257061b
4
+ data.tar.gz: 64fcbee6db3ccb50472b4ab545ab56e06279a624
5
5
  SHA512:
6
- metadata.gz: 0fe919474cdc8d4e31f7ed8703600d960610876613ffca815d79267cf00fdc02e1dfdc5af2e692ae72d0005e67e9372f24da289c4abb1806f09abb06ad8c6f80
7
- data.tar.gz: 590ccdd14141ef0feeb4482a8063d634067f9b3ad73d239a2afe0782be6857fa99bc1751a7c2508957e3cf53782c320942eee535899ca6d24a2e69e6ab86ca7f
6
+ metadata.gz: e83e46bef67b3dc054aef46e7cd5f3e615508b2bf142f770b243393039c410110994a95a239d20b8a4bef51e68cd5aefbef052e1c0459b2a9e15e265052ecdef
7
+ data.tar.gz: e809c1bb0881fb6717ccacea4b76981debeb53371a82b0aa20043040fec3b632049ccab763de15b8ffebf0830e772eac37e9094df437da9b829a16d00b3a34b9
data/Readme.md CHANGED
@@ -19,10 +19,10 @@ Just call for help!
19
19
 
20
20
  $ java -jar anschel-1.2.3.jar help
21
21
  Commands:
22
- anschel agent # Run the Anschel agent
23
- anschel art # View the application art
22
+ anschel agent # Run application
23
+ anschel art # Show application art
24
24
  anschel help [COMMAND] # Describe available commands or one specific command
25
- anschel version # Echo the application version
25
+ anschel version # Show application version
26
26
 
27
27
  Probably you're most interested in the `agent` command:
28
28
 
@@ -31,12 +31,15 @@ Probably you're most interested in the `agent` command:
31
31
  anschel agent
32
32
 
33
33
  Options:
34
- -c, [--config=CONFIG] # Main configuration file
34
+ -c, [--config=CONFIG] # Path to primary configuration file
35
35
  # Default: /etc/anschel.json
36
+ -i, [--stats-interval=N] # Interval for reporting stats (seconds)
37
+ # Default: 30
36
38
  -L, [--log=LOG] # Log to file instead of STDOUT
37
39
  -V, [--debug], [--no-debug] # Enable DEBUG-level logging
40
+ -Z, [--trace], [--no-trace] # Enable TRACE-level logging (precedence over DEBUG)
38
41
 
39
- Run the Anschel agent
42
+ Run application
40
43
 
41
44
 
42
45
 
@@ -137,6 +140,8 @@ You might deploy Anschel with Upstart. Here's a minimal config:
137
140
 
138
141
  ### Changelog
139
142
 
140
- #### v1.0 (devlop)
143
+ #### v1.0
144
+
145
+ _In devlopment_
141
146
 
142
147
  - Intial implementation of the Kafka-to-Elasticsearch pipeline
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.0
1
+ 0.2.0
@@ -6,7 +6,8 @@
6
6
  # }
7
7
  module Anschel
8
8
  class Filter
9
- def convert conf
9
+ def convert conf, log
10
+ log.debug event: 'compile-filter', filter: 'convert', conf: conf
10
11
  field = conf.delete :field
11
12
  type = conf.delete :type
12
13
 
@@ -21,6 +22,9 @@ module Anschel
21
22
  'string' => :to_s
22
23
  }
23
24
 
25
+ log.debug event: 'compiled-filter', filter: 'convert', \
26
+ field: field, type: type
27
+
24
28
  lambda do |event|
25
29
  return event unless event.has_key? field
26
30
  event[field] = event[field].send type_conversions[type]
@@ -7,7 +7,7 @@
7
7
  # }
8
8
  module Anschel
9
9
  class Filter
10
- def gsub conf
10
+ def gsub conf, log
11
11
  field = conf.delete :field
12
12
  match = Regexp.new conf.delete(:match)
13
13
  replace = conf.delete :replace
@@ -18,6 +18,10 @@ module Anschel
18
18
 
19
19
  field = field.to_sym
20
20
 
21
+
22
+ log.debug event: 'compiled-filter', filter: 'gsub', \
23
+ field: field, match: match, replace: replace
24
+
21
25
  lambda do |event|
22
26
  return event unless event.has_key? field
23
27
  event[field].gsub! match, replace
@@ -1,6 +1,13 @@
1
+ # {
2
+ # "index": {
3
+ # "stamp": "",
4
+ # "prefix": "",
5
+ # "suffix": ""
6
+ # }
7
+ # }
1
8
  module Anschel
2
9
  class Filter
3
- def index conf
10
+ def index conf, log
4
11
  stamp = conf.delete(:stamp) || '@timestamp'
5
12
  prefix = conf.delete(:prefix) || 'logs-%{type}-'
6
13
  suffix = conf.delete(:suffix) || '%Y.%m.%d'
@@ -13,6 +20,10 @@ module Anschel
13
20
  joda = joda.withDefaultYear(Time.new.year)
14
21
  joda = joda.withOffsetParsed
15
22
 
23
+
24
+ log.debug event: 'compiled-filter', filter: 'index', \
25
+ stamp: stamp, prefix: prefix, suffix: suffix
26
+
16
27
  lambda do |event|
17
28
  return event unless event.has_key? stamp
18
29
  millis = joda.parseMillis event[stamp]
@@ -6,7 +6,7 @@
6
6
  # }
7
7
  module Anschel
8
8
  class Filter
9
- def parse conf
9
+ def parse conf, log
10
10
  field = conf.delete :field
11
11
  pattern = Regexp.new conf.delete(:pattern)
12
12
 
@@ -15,6 +15,10 @@ module Anschel
15
15
 
16
16
  field = field.to_sym
17
17
 
18
+
19
+ log.debug event: 'compiled-filter', filter: 'parse', \
20
+ field: field, pattern: pattern
21
+
18
22
  lambda do |event|
19
23
  return event unless event.has_key? field
20
24
  mdata = pattern.match event[field]
@@ -7,7 +7,7 @@
7
7
  # }
8
8
  module Anschel
9
9
  class Filter
10
- def scan conf
10
+ def scan conf, log
11
11
  field = conf.delete :field
12
12
  pattern = Regexp.new conf.delete(:pattern)
13
13
  target = conf.delete :target
@@ -19,6 +19,10 @@ module Anschel
19
19
  field = field.to_sym
20
20
  target = target.to_sym
21
21
 
22
+
23
+ log.debug event: 'compiled-filter', filter: 'scan', \
24
+ field: field, pattern: pattern, target: target
25
+
22
26
  lambda do |event|
23
27
  return event unless event.has_key? field
24
28
  results = event[field].scan(pattern).flatten.uniq
@@ -6,19 +6,17 @@
6
6
  # }
7
7
  module Anschel
8
8
  class Filter
9
- def stamp conf
9
+ def stamp conf, log
10
10
  field = conf.delete :field
11
11
  pattern = conf.delete :pattern
12
12
  target = conf.delete :target
13
- precision = conf.delete :precision
14
13
  error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'stamp-error'
15
14
 
16
15
  raise 'Missing required "field" for "stamp" filter' if field.nil?
17
16
  raise 'Missing required "pattern" for "stamp" filter' if pattern.nil?
18
17
 
19
- patterns = pattern.is_a?(Array) ? pattern : [ pattern ]
20
- precision = (precision || 3).to_i
21
- target ||= '@timestamp'
18
+ patterns = pattern.is_a?(Array) ? pattern : [ pattern ]
19
+ target ||= '@timestamp'
22
20
 
23
21
  field = field.to_sym
24
22
  target = target.to_sym
@@ -29,12 +27,16 @@ module Anschel
29
27
  joda = joda.withOffsetParsed
30
28
  end
31
29
 
30
+
31
+ log.debug event: 'compiled-filter', filter: 'stamp', \
32
+ field: field, pattern: pattern, target: target, error_tag: error_tag
33
+
32
34
  lambda do |event|
33
35
  return event unless event.has_key? field
34
36
  parsers.each do |joda|
35
37
  millis = joda.parseMillis event[field]
36
38
  begin
37
- event[target] = Time.at(0.001 * millis).iso8601(precision)
39
+ event[target] = Time.at(0.001 * millis).iso8601(3)
38
40
  return filtered(event, conf)
39
41
  rescue
40
42
  end
@@ -11,13 +11,14 @@ module Anschel
11
11
 
12
12
  attr_reader :filters
13
13
 
14
- def initialize config, log
14
+ def initialize config, stats, log
15
+ log.trace event: 'filter', config: config
15
16
  @filters = Hash.new { |h,k| h[k] = [] }
16
17
  config.each do |event_type, filter_defns|
17
18
  filter_defns.each do |filter_defn|
18
19
  filter_type = filter_defn.keys.first
19
20
  filter_conf = filter_defn[filter_type]
20
- @filters[event_type] << self.send(filter_type, filter_conf)
21
+ @filters[event_type] << self.send(filter_type, filter_conf, log)
21
22
  end
22
23
  end
23
24
  log.info event: 'filter-loaded'
data/lib/anschel/input.rb CHANGED
@@ -3,7 +3,8 @@ require 'jruby-kafka'
3
3
 
4
4
  module Anschel
5
5
  class Input
6
- def initialize config, log
6
+ def initialize config, stats, log
7
+ log.trace event: 'input', config: config
7
8
  qsize = config.delete(:queue_size) || 1000
8
9
  @queue = SizedQueue.new qsize
9
10
  consumer_group = Kafka::Group.new config
@@ -11,7 +12,7 @@ module Anschel
11
12
 
12
13
  trap('SIGINT') do
13
14
  consumer_group.shutdown
14
- log.info event: 'goodbye'
15
+ log.info event: 'goodbye', version: VERSION
15
16
  exit
16
17
  end
17
18
 
data/lib/anschel/main.rb CHANGED
@@ -5,7 +5,7 @@ require 'jrjackson'
5
5
 
6
6
  require_relative 'mjolnir'
7
7
  require_relative 'metadata'
8
-
8
+ require_relative 'stats'
9
9
  require_relative 'input'
10
10
  require_relative 'filter'
11
11
  require_relative 'output'
@@ -15,37 +15,42 @@ module Anschel
15
15
  class Main < Mjolnir
16
16
 
17
17
 
18
- desc 'version', 'Echo the application version'
18
+ desc 'version', 'Show application version'
19
19
  def version
20
20
  puts VERSION
21
21
  end
22
22
 
23
23
 
24
- desc 'art', 'View the application art'
24
+ desc 'art', 'Show application art'
25
25
  def art
26
26
  puts "\n%s\n" % ART
27
27
  end
28
28
 
29
29
 
30
- desc 'agent', 'Run the Anschel agent'
30
+ desc 'agent', 'Run application'
31
31
  option :config, \
32
32
  type: :string,
33
33
  aliases: %w[ -c ],
34
- desc: 'Main configuration file',
34
+ desc: 'Path to primary configuration file',
35
35
  default: '/etc/anschel.json'
36
+ option :stats_interval, \
37
+ type: :numeric,
38
+ aliases: %w[ -i ],
39
+ desc: 'Interval for reporting stats (seconds)',
40
+ default: 30
36
41
  include_common_options
37
42
  def agent
38
- log.info event: 'hello'
43
+ log.info event: 'hello', version: VERSION, options: options.to_hash
39
44
  config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
40
45
  setup_log4j config[:log4j]
41
46
 
42
- input = Input.new config[:kafka], log
43
- filter = Filter.new config[:filter], log
44
- output = Output.new config[:elasticsearch], log
47
+ stats = Stats.new log, options.stats_interval
48
+ input = Input.new config[:kafka], stats, log
49
+ filter = Filter.new config[:filter], stats, log
50
+ output = Output.new config[:elasticsearch], stats, log
45
51
 
46
- start = Time.now
47
- count = 0
48
- sample = 100_000
52
+ stats.create 'count', 0
53
+ stats.get 'count'
49
54
 
50
55
  ts = num_cpus.times.map do
51
56
  Thread.new do
@@ -53,17 +58,7 @@ module Anschel
53
58
  event = JrJackson::Json.load \
54
59
  input.shift.message.to_s, symbolize_keys: true
55
60
  output.push filter.apply(event)
56
- if (count += 1) % sample == 0
57
- old_count, now = count, Time.now
58
- elapsed, start = (now - start).to_f, now
59
- rate = 1.0 * sample / elapsed
60
- log.info \
61
- event: 'stat',
62
- count: old_count,
63
- sample: sample,
64
- elapsed_s: elapsed,
65
- rate_eps: rate
66
- end
61
+ stats.inc 'count'
67
62
  end
68
63
  end
69
64
  end
@@ -21,15 +21,16 @@ module Anschel
21
21
  # Pull the project version out of the VERSION file
22
22
  VERSION = File.read(File.join(ROOT, 'VERSION')).strip
23
23
 
24
+ # Thank you, http://patorjk.com/software/taag
24
25
  ART = <<-'EOART'
25
- 888 888
26
- 888 888
27
- 888 888
28
- 8888b. 88888b. .d8888b .d8888b 88888b. .d88b. 888
29
- "88b 888 "88b 88K d88P" 888 "88b d8P Y8b 888
30
- .d888888 888 888 "Y8888b. 888 888 888 88888888 888
31
- 888 888 888 888 X88 Y88b. 888 888 Y8b. 888
32
- "Y888888 888 888 88888P' "Y8888P 888 888 "Y8888 888
26
+
27
+ oooo oooo
28
+ `888 `888
29
+ .oooo. ooo. .oo. .oooo.o .ooooo. 888 .oo. .ooooo. 888
30
+ `P )88b `888P"Y88b d88( "8 d88' `"Y8 888P"Y88b d88' `88b 888
31
+ .oP"888 888 888 `"Y88b. 888 888 888 888ooo888 888
32
+ d8( 888 888 888 o. )88b 888 .o8 888 888 888 .o 888
33
+ `Y888""8o o888o o888o 8""888P' `Y8bod8P' o888o o888o `Y8bod8P' o888o
33
34
 
34
35
  EOART
35
36
  end
@@ -1,4 +1,5 @@
1
1
  require 'thor'
2
+ require 'slog'
2
3
 
3
4
 
4
5
  module Anschel
@@ -19,6 +20,12 @@ module Anschel
19
20
  aliases: %w[ -V ],
20
21
  desc: 'Enable DEBUG-level logging',
21
22
  default: ENV['ANSCHEL_DEBUG'] || false
23
+ },
24
+ trace: {
25
+ type: :boolean,
26
+ aliases: %w[ -Z ],
27
+ desc: 'Enable TRACE-level logging (precedence over DEBUG)',
28
+ default: ENV['ANSCHEL_TRACE'] || false
22
29
  }
23
30
  }
24
31
 
@@ -35,8 +42,9 @@ module Anschel
35
42
  # Construct a Logger given the command-line options
36
43
  def log
37
44
  return @logger if defined? @logger
38
- @logger = Logger.new(options.log || STDOUT)
39
- @logger.level = Logger::DEBUG if options.debug?
45
+ @logger = Slog.new out: (options.log || $stdout), prettify: false
46
+ @logger.level = :debug if options.debug?
47
+ @logger.level = :trace if options.trace?
40
48
  @logger
41
49
  end
42
50
 
@@ -7,11 +7,12 @@ require 'elasticsearch'
7
7
 
8
8
  module Anschel
9
9
  class Output
10
- def initialize config, log
10
+ def initialize config, stats, log
11
+ log.trace event: 'output', config: config
11
12
  pattern = config.delete(:index_pattern)
12
13
  qsize = config.delete(:queue_size) || 2000
13
14
  bsize = config.delete(:bulk_size) || 500
14
- timeout = config.delete(:bulk_timeout) || 1.0
15
+ timeout = config.delete(:bulk_timeout) || 0.5
15
16
  slice = timeout / bsize
16
17
  client = Elasticsearch::Client.new config
17
18
  client.transport.reload_connections!
@@ -0,0 +1,88 @@
1
+ require 'thread'
2
+ require 'logger'
3
+
4
+
5
+ module Anschel
6
+ class Stats
7
+
8
+ def initialize logger, interval=60
9
+ @logger = logger
10
+ @interval = interval
11
+ @stats = Hash.new
12
+ @lock = Mutex.new
13
+
14
+ Thread.new do
15
+ loop do
16
+ sleep @interval
17
+ report_and_reset
18
+ end
19
+ end
20
+
21
+ @logger.info event: 'stats-loaded'
22
+ end
23
+
24
+
25
+ def create name, default=nil
26
+ with_lock do
27
+ stats[name] = Hash.new { |h,k| h[k] = default }
28
+ end
29
+ end
30
+
31
+ def delete name
32
+ with_lock do
33
+ stats.delete name
34
+ end
35
+ end
36
+
37
+ def inc name, by=1
38
+ with_lock do
39
+ stats[name][:val] += by
40
+ end
41
+ end
42
+
43
+ def dec name, by=1
44
+ with_lock do
45
+ stats[name][:val] -= by
46
+ end
47
+ end
48
+
49
+ def set name, to
50
+ with_lock do
51
+ stats[name][:val] = to
52
+ end
53
+ end
54
+
55
+ def get name
56
+ with_lock do
57
+ stats[name][:val]
58
+ end
59
+ end
60
+
61
+
62
+ private
63
+ attr_reader :stats
64
+
65
+ def with_lock &block
66
+ @lock.synchronize do
67
+ yield
68
+ end
69
+ end
70
+
71
+ def report_and_reset
72
+ ready_stats = with_lock do
73
+ stats.keys.flat_map do |k|
74
+ v = stats[k].delete(:val)
75
+ [
76
+ [ k, v ],
77
+ [ "#{k}_rate", 1.0 * v / @interval ]
78
+ ]
79
+ end
80
+ end
81
+
82
+ @logger.info \
83
+ event: 'stats',
84
+ interval: @interval,
85
+ stats: Hash[ready_stats]
86
+ end
87
+ end
88
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: anschel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Clemmer
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - '>='
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ requirement: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '>='
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ name: slog
90
+ prerelease: false
91
+ type: :runtime
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '>='
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  description: Logstash-like for moving events from Kafka into Elasticsearch.
84
98
  email: sczizzo@gmail.com
85
99
  executables:
@@ -104,6 +118,7 @@ files:
104
118
  - lib/anschel/metadata.rb
105
119
  - lib/anschel/mjolnir.rb
106
120
  - lib/anschel/output.rb
121
+ - lib/anschel/stats.rb
107
122
  homepage: https://github.com/sczizzo/anschel
108
123
  licenses:
109
124
  - ISC