anschel 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2c9d437249846bfe1e7ab107fb86e8338a709d8d
4
+ data.tar.gz: 75e4124f0cd424865aa63692d2fdb16001e9bb58
5
+ SHA512:
6
+ metadata.gz: 61fd41165f874a2aa55677b3d3a562994e9283d67c949a2d8945df295e1a9d9554af508ce2c39e05b781b03dc8f8f562de1ef3d4c5af8da23c67084ef375bccd
7
+ data.tar.gz: 4bb4b827b537644b7f5495a42dccf79d867e8f0a76c8cfc17c5abd2df1b91758c8f1271d8147854a2a56871ba2d34c76afa0940ec7b6d6834fca6fafb9110962
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2015 Sean Clemmer and Blue Jeans Network
2
+
3
+ Permission to use, copy, modify, and/or distribute this software for any
4
+ purpose with or without fee is hereby granted, provided that the above
5
+ copyright notice and this permission notice appear in all copies.
6
+
7
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
8
+ REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
9
+ AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
10
+ INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
11
+ LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
12
+ OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
13
+ PERFORMANCE OF THIS SOFTWARE.
data/Readme.md ADDED
@@ -0,0 +1,3 @@
1
+ # Anschel
2
+
3
+ Companion to Franz
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.2
data/bin/anschel ADDED
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env ruby
2
+ require 'anschel'
3
+ Anschel::Main.start
@@ -0,0 +1,29 @@
1
+ # {
2
+ # "convert": {
3
+ # "field": "duration",
4
+ # "type": "integer"
5
+ # }
6
+ # }
7
+ module Anschel
8
+ class Filter
9
+ def convert conf
10
+ field = conf.delete :field
11
+ type = conf.delete :type
12
+
13
+ raise 'Missing required "field" for "convert" filter' if field.nil?
14
+ raise 'Missing required "type" for "convert" filter' if type.nil?
15
+
16
+ type_conversions = {
17
+ 'integer' => :to_i,
18
+ 'float' => :to_f,
19
+ 'string' => :to_s
20
+ }
21
+
22
+ lambda do |event|
23
+ return event unless event.has_key? field
24
+ event[field] = event[field].send type_conversions[type]
25
+ filtered event, conf
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,26 @@
1
+ # {
2
+ # "gsub": {
3
+ # "field": "",
4
+ # "match": "",
5
+ # "replace": ""
6
+ # }
7
+ # }
8
+ module Anschel
9
+ class Filter
10
+ def gsub conf
11
+ field = conf.delete :field
12
+ match = Regexp.new conf.delete(:match)
13
+ replace = conf.delete :replace
14
+
15
+ raise 'Missing required "field" for "gsub" filter' if field.nil?
16
+ raise 'Missing required "match" for "gsub" filter' if match.nil?
17
+ raise 'Missing required "replace" for "gsub" filter' if replace.nil?
18
+
19
+ lambda do |event|
20
+ return event unless event.has_key? field
21
+ event[field].gsub! match, replace
22
+ filtered event, conf
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,26 @@
1
+ # {
2
+ # "parse": {
3
+ # "field": "message",
4
+ # "pattern": "()"
5
+ # }
6
+ # }
7
+ module Anschel
8
+ class Filter
9
+ def parse conf
10
+ field = conf.delete :field
11
+ pattern = Regexp.new conf.delete(:pattern)
12
+
13
+ raise 'Missing required "field" for "parse" filter' if field.nil?
14
+ raise 'Missing required "pattern" for "parse" filter' if pattern.nil?
15
+
16
+ lambda do |event|
17
+ return event unless event.has_key? field
18
+ mdata = pattern.match event[field]
19
+ mdata.names.each do |group|
20
+ event[group] = mdata[group]
21
+ end
22
+ filtered event, conf
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,33 @@
1
+ # {
2
+ # "scan": {
3
+ # "field": "",
4
+ # "pattern": "",
5
+ # "target": ""
6
+ # }
7
+ # }
8
+ module Anschel
9
+ class Filter
10
+ def scan conf
11
+ field = conf.delete :field
12
+ pattern = Regexp.new conf.delete(:pattern)
13
+ target = conf.delete :target
14
+
15
+ raise 'Missing required "field" for "scan" filter' if field.nil?
16
+ raise 'Missing required "pattern" for "scan" filter' if pattern.nil?
17
+ raise 'Missing required "target" for "convert" filter' if target.nil?
18
+
19
+ lambda do |event|
20
+ return event unless event.has_key? field
21
+ results = event[field].scan(pattern).flatten.uniq
22
+
23
+ if results.empty?
24
+ event
25
+ else
26
+ event[target] ||= []
27
+ event[target] += results
28
+ filtered event, conf
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,48 @@
1
+ # {
2
+ # "stamp": {
3
+ # "field": "timestamp",
4
+ # "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
5
+ # }
6
+ # }
7
+ module Anschel
8
+ class Filter
9
+ def stamp conf
10
+ field = conf.delete :field
11
+ pattern = conf.delete :pattern
12
+ target = conf.delete :target
13
+ precision = conf.delete :precision
14
+ error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'stamp-error'
15
+
16
+ raise 'Missing required "field" for "stamp" filter' if field.nil?
17
+ raise 'Missing required "pattern" for "stamp" filter' if pattern.nil?
18
+
19
+ patterns = pattern.is_a?(Array) ? pattern : [ pattern ]
20
+ precision = (precision || 3).to_i
21
+ target ||= '@timestamp'
22
+
23
+ parsers = patterns.map do |p|
24
+ joda = org.joda.time.format.DateTimeFormat.forPattern(p)
25
+ joda = joda.withDefaultYear(Time.new.year)
26
+ joda = joda.withOffsetParsed
27
+ end
28
+
29
+ lambda do |event|
30
+ return event unless event.has_key? field
31
+ parsers.each do |joda|
32
+ millis = joda.parseMillis event[field]
33
+ begin
34
+ event[target] = Time.at(0.001 * millis).iso8601(precision)
35
+ return filtered(event, conf)
36
+ rescue
37
+ end
38
+ end
39
+
40
+ if error_tag
41
+ event['tags'] ||= []
42
+ event['tags'] << error_tag
43
+ end
44
+ event
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,41 @@
1
+ require_relative 'filter/convert'
2
+ require_relative 'filter/gsub'
3
+ require_relative 'filter/parse'
4
+ require_relative 'filter/scan'
5
+ require_relative 'filter/stamp'
6
+
7
+
8
+ module Anschel
9
+ class Filter
10
+
11
+ attr_reader :filters
12
+
13
+ def initialize config
14
+ @filters = Hash.new { |h,k| h[k] = [] }
15
+ config.each do |event_type, filter_defns|
16
+ filter_defns.each do |filter_defn|
17
+ filter_type = filter_defn.keys.first
18
+ filter_conf = filter_defn[filter_type]
19
+ @filters[event_type] << self.send(filter_type, filter_conf)
20
+ end
21
+ end
22
+ end
23
+
24
+
25
+ def apply event
26
+ raise 'Event does not have a "type" field' unless event['type']
27
+ filters[:*].each { |f| f.call event }
28
+ filters[event[:type]].each { |f| f.call event }
29
+ event
30
+ end
31
+
32
+
33
+ def filtered event, options
34
+ if remove_field = options[:remove_field]
35
+ event.delete remove_field
36
+ end
37
+ event
38
+ end
39
+
40
+ end
41
+ end
@@ -0,0 +1,21 @@
1
+ require 'jruby-kafka'
2
+
3
+
4
+ module Anschel
5
+ class Input
6
+ def initialize config
7
+ qsize = config.delete(:queue_size) || 1000
8
+ @queue = SizedQueue.new qsize
9
+ consumer_group = Kafka::Group.new config
10
+ consumer_group.run num_cpus, @queue
11
+
12
+ trap('SIGINT') do
13
+ consumer_group.shutdown
14
+ exit
15
+ end
16
+ end
17
+
18
+
19
+ def pop ; @queue.pop end
20
+ end
21
+ end
@@ -0,0 +1,70 @@
1
+ require 'logger'
2
+ require 'tempfile'
3
+
4
+ require 'jrjackson'
5
+
6
+ require_relative 'mjolnir'
7
+ require_relative 'metadata'
8
+
9
+ require_relative 'input'
10
+ require_relative 'filter'
11
+ require_relative 'output'
12
+
13
+
14
+ module Anschel
15
+ class Main < Mjolnir
16
+
17
+
18
+ desc 'version', 'Echo the application version'
19
+ def version
20
+ puts VERSION
21
+ end
22
+
23
+
24
+ desc 'art', 'View the application art'
25
+ def art
26
+ puts "\n%s\n" % ART
27
+ end
28
+
29
+
30
+ desc 'test', 'Test the Anschel agent'
31
+ option :config, \
32
+ type: :string,
33
+ aliases: %w[ -c ],
34
+ desc: 'Main configuration file',
35
+ default: '/etc/anschel.json'
36
+ include_common_options
37
+ def test
38
+ config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
39
+ setup_log4j config[:log4j]
40
+
41
+ input = Input.new config[:kafka]
42
+ filter = Filter.new config[:filter]
43
+ output = Output.new config[:elasticsearch]
44
+
45
+ start = Time.now
46
+ count = 0
47
+
48
+ ts = num_cpus.times.map do
49
+ Thread.new do
50
+ loop do
51
+ event = JrJackson::Json.load input.pop.message.to_s
52
+ output.push filter.apply(event)
53
+ if (count += 1) % 100_000 == 0
54
+ elapsed = Time.now - start
55
+ log.info \
56
+ event: 'stat',
57
+ count: count,
58
+ elapsed_s: elapsed.to_f,
59
+ rate_eps: ( 1.0 * count / elapsed.to_f )
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ ts.map &:join
66
+ exit
67
+ end
68
+
69
+ end
70
+ end
@@ -0,0 +1,36 @@
1
+ module Anschel
2
+ NAME = 'anschel'
3
+
4
+ # A quick summary for use in the command-line interface
5
+ SUMMARY = %q.Companion to Franz.
6
+
7
+ # Take credit for your work
8
+ AUTHOR = 'Sean Clemmer'
9
+
10
+ # Take responsibility for your work
11
+ EMAIL = 'sczizzo@gmail.com'
12
+
13
+ # Like the MIT license, but even simpler
14
+ LICENSE = 'ISC'
15
+
16
+ # Where you should look first
17
+ HOMEPAGE = 'https://github.com/sczizzo/anschel'
18
+
19
+ # Project root
20
+ ROOT = File.join File.dirname(__FILE__), '..', '..'
21
+
22
+ # Pull the project version out of the VERSION file
23
+ VERSION = File.read(File.join(ROOT, 'VERSION')).strip
24
+
25
+ ART = <<-'EOART'
26
+ 888 888
27
+ 888 888
28
+ 888 888
29
+ 8888b. 88888b. .d8888b .d8888b 88888b. .d88b. 888
30
+ "88b 888 "88b 88K d88P" 888 "88b d8P Y8b 888
31
+ .d888888 888 888 "Y8888b. 888 888 888 88888888 888
32
+ 888 888 888 888 X88 Y88b. 888 888 Y8b. 888
33
+ "Y888888 888 888 88888P' "Y8888P 888 888 "Y8888 888
34
+
35
+ EOART
36
+ end
@@ -0,0 +1,44 @@
1
+ require 'thor'
2
+
3
+
4
+ module Anschel
5
+
6
+ # Thor's hammer! Like Thor with better logging
7
+ class Mjolnir < Thor
8
+
9
+ # Common options for Thor commands
10
+ COMMON_OPTIONS = {
11
+ log: {
12
+ type: :string,
13
+ aliases: %w[ -L ],
14
+ desc: 'Log to file instead of STDOUT',
15
+ default: ENV['ANSCHEL_LOG'] || nil
16
+ },
17
+ debug: {
18
+ type: :boolean,
19
+ aliases: %w[ -V ],
20
+ desc: 'Enable DEBUG-level logging',
21
+ default: ENV['ANSCHEL_DEBUG'] || false
22
+ }
23
+ }
24
+
25
+ # Decorate Thor commands with the options above
26
+ def self.include_common_options
27
+ COMMON_OPTIONS.each do |name, spec|
28
+ option name, spec
29
+ end
30
+ end
31
+
32
+
33
+ no_commands do
34
+
35
+ # Construct a Logger given the command-line options
36
+ def log
37
+ @logger = Logger.new(options.log || STDOUT)
38
+ @logger.level = Logger::DEBUG if options.debug?
39
+ @logger
40
+ end
41
+
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,30 @@
1
+ require 'elasticsearch'
2
+
3
+
4
+ module Anschel
5
+ class Output
6
+ def initialize config
7
+ pattern = config.delete(:index_pattern)
8
+ bsize = config.delete(:bulk_size) || 500
9
+ qsize = config.delete(:queue_size) || 2000
10
+ @queue = SizedQueue.new qsize
11
+ client = Elasticsearch::Client.new config
12
+ client.transport.reload_connections!
13
+
14
+ Thread.new do
15
+ loop do
16
+ events = bsize.times.map { @queue.pop }
17
+
18
+ body = events.map do |e|
19
+ index = e.fetch '@@anschel_index', 'logs-anschel'
20
+ { index: { _index: index, _type: e['type'], data: e } }
21
+ end
22
+
23
+ client.bulk body: body
24
+ end
25
+ end
26
+ end
27
+
28
+ def push event ; @queue << event end
29
+ end
30
+ end
data/lib/anschel.rb ADDED
@@ -0,0 +1,27 @@
1
+ # Return the number of CPU cores (should work on Unixy platforms)
2
+ def num_cpus
3
+ return Java::Java.lang.Runtime.getRuntime.availableProcessors if defined? Java::Java
4
+ return File.read('/proc/cpuinfo').scan(/^processor\s*:/).size if File.exist? '/proc/cpuinfo'
5
+ return `sysctl -a | grep cpu`.split(/\s+/,2).last.to_i
6
+ rescue
7
+ return 1
8
+ end
9
+
10
+ def setup_log4j config
11
+ path = config.delete(:path) || '/var/log/anschel4j.log'
12
+ pattern = config.delete(:pattern) || '[%d] %p %m (%c)%n'
13
+ Tempfile.open('anschel_log4j') do |f|
14
+ log4j = %Q|
15
+ log4j.rootLogger=INFO, A1
16
+ log4j.appender.A1=org.apache.log4j.RollingFileAppender
17
+ log4j.appender.A1.File=%s
18
+ log4j.appender.A1.layout=org.apache.log4j.PatternLayout
19
+ log4j.appender.A1.layout.ConversionPattern=%s
20
+ | % [ path, pattern ]
21
+ f.write log4j.gsub(/^\s+/,'')
22
+ f.rewind
23
+ org.apache.log4j.PropertyConfigurator.configure(f.path)
24
+ end
25
+ end
26
+
27
+ require_relative 'anschel/main'
metadata ADDED
@@ -0,0 +1,116 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: anschel
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: ruby
6
+ authors:
7
+ - Sean Clemmer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-06-21 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - '>='
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ name: thor
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ name: jruby-kafka
34
+ prerelease: false
35
+ type: :runtime
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '>='
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ name: jrjackson
48
+ prerelease: false
49
+ type: :runtime
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - '>='
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ name: elasticsearch
62
+ prerelease: false
63
+ type: :runtime
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: Companion to Franz.
70
+ email: sczizzo@gmail.com
71
+ executables:
72
+ - anschel
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - LICENSE
77
+ - Readme.md
78
+ - VERSION
79
+ - bin/anschel
80
+ - lib/anschel.rb
81
+ - lib/anschel/filter.rb
82
+ - lib/anschel/filter/convert.rb
83
+ - lib/anschel/filter/gsub.rb
84
+ - lib/anschel/filter/parse.rb
85
+ - lib/anschel/filter/scan.rb
86
+ - lib/anschel/filter/stamp.rb
87
+ - lib/anschel/input.rb
88
+ - lib/anschel/main.rb
89
+ - lib/anschel/metadata.rb
90
+ - lib/anschel/mjolnir.rb
91
+ - lib/anschel/output.rb
92
+ homepage: https://github.com/sczizzo/anschel
93
+ licenses:
94
+ - ISC
95
+ metadata: {}
96
+ post_install_message:
97
+ rdoc_options: []
98
+ require_paths:
99
+ - lib
100
+ required_ruby_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '>='
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ required_rubygems_version: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ requirements: []
111
+ rubyforge_project:
112
+ rubygems_version: 2.4.6
113
+ signing_key:
114
+ specification_version: 4
115
+ summary: Companion to Franz
116
+ test_files: []