anschel 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c9d437249846bfe1e7ab107fb86e8338a709d8d
+  data.tar.gz: 75e4124f0cd424865aa63692d2fdb16001e9bb58
+SHA512:
+  metadata.gz: 61fd41165f874a2aa55677b3d3a562994e9283d67c949a2d8945df295e1a9d9554af508ce2c39e05b781b03dc8f8f562de1ef3d4c5af8da23c67084ef375bccd
+  data.tar.gz: 4bb4b827b537644b7f5495a42dccf79d867e8f0a76c8cfc17c5abd2df1b91758c8f1271d8147854a2a56871ba2d34c76afa0940ec7b6d6834fca6fafb9110962

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2015 Sean Clemmer and Blue Jeans Network
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

data/Readme.md

@@ -0,0 +1,3 @@
+# Anschel
+
+Companion to Franz

data/VERSION

@@ -0,0 +1 @@
+0.0.2

data/bin/anschel

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'anschel'
+Anschel::Main.start

data/lib/anschel/filter/convert.rb

@@ -0,0 +1,29 @@
+# {
+#   "convert": {
+#     "field": "duration",
+#     "type": "integer"
+#   }
+# }
+module Anschel
+  class Filter
+    def convert conf
+      field = conf.delete :field
+      type  = conf.delete :type
+
+      raise 'Missing required "field" for "convert" filter' if field.nil?
+      raise 'Missing required "type" for "convert" filter' if type.nil?
+
+      type_conversions = {
+        'integer' => :to_i,
+        'float'   => :to_f,
+        'string'  => :to_s
+      }
+
+      lambda do |event|
+        return event unless event.has_key? field
+        event[field] = event[field].send type_conversions[type]
+        filtered event, conf
+      end
+    end
+  end
+end

data/lib/anschel/filter/gsub.rb

@@ -0,0 +1,26 @@
+# {
+#   "gsub": {
+#     "field": "",
+#     "match": "",
+#     "replace": ""
+#   }
+# }
+module Anschel
+  class Filter
+    def gsub conf
+      field   = conf.delete :field
+      match   = Regexp.new conf.delete(:match)
+      replace = conf.delete :replace
+
+      raise 'Missing required "field" for "gsub" filter' if field.nil?
+      raise 'Missing required "match" for "gsub" filter' if match.nil?
+      raise 'Missing required "replace" for "gsub" filter' if replace.nil?
+
+      lambda do |event|
+        return event unless event.has_key? field
+        event[field].gsub! match, replace
+        filtered event, conf
+      end
+    end
+  end
+end

data/lib/anschel/filter/parse.rb

@@ -0,0 +1,26 @@
+# {
+#   "parse": {
+#     "field": "message",
+#     "pattern": "()"
+#   }
+# }
+module Anschel
+  class Filter
+    def parse conf
+      field   = conf.delete :field
+      pattern = Regexp.new conf.delete(:pattern)
+
+      raise 'Missing required "field" for "parse" filter' if field.nil?
+      raise 'Missing required "pattern" for "parse" filter' if pattern.nil?
+
+      lambda do |event|
+        return event unless event.has_key? field
+        mdata = pattern.match event[field]
+        mdata.names.each do |group|
+          event[group] = mdata[group]
+        end
+        filtered event, conf
+      end
+    end
+  end
+end

data/lib/anschel/filter/scan.rb

@@ -0,0 +1,33 @@
+# {
+#   "scan": {
+#     "field": "",
+#     "pattern": "",
+#     "target": ""
+#   }
+# }
+module Anschel
+  class Filter
+    def scan conf
+      field   = conf.delete :field
+      pattern = Regexp.new conf.delete(:pattern)
+      target  = conf.delete :target
+
+      raise 'Missing required "field" for "scan" filter' if field.nil?
+      raise 'Missing required "pattern" for "scan" filter' if pattern.nil?
+      raise 'Missing required "target" for "convert" filter' if target.nil?
+
+      lambda do |event|
+        return event unless event.has_key? field
+        results = event[field].scan(pattern).flatten.uniq
+
+        if results.empty?
+          event
+        else
+          event[target] ||= []
+          event[target]  += results
+          filtered event, conf
+        end
+      end
+    end
+  end
+end

data/lib/anschel/filter/stamp.rb

@@ -0,0 +1,48 @@
+# {
+#   "stamp": {
+#     "field": "timestamp",
+#     "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
+#   }
+# }
+module Anschel
+  class Filter
+    def stamp conf
+      field     = conf.delete :field
+      pattern   = conf.delete :pattern
+      target    = conf.delete :target
+      precision = conf.delete :precision
+      error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'stamp-error'
+
+      raise 'Missing required "field" for "stamp" filter' if field.nil?
+      raise 'Missing required "pattern" for "stamp" filter' if pattern.nil?
+
+      patterns  = pattern.is_a?(Array) ? pattern : [ pattern ]
+      precision = (precision || 3).to_i
+      target  ||= '@timestamp'
+
+      parsers = patterns.map do |p|
+        joda = org.joda.time.format.DateTimeFormat.forPattern(p)
+        joda = joda.withDefaultYear(Time.new.year)
+        joda = joda.withOffsetParsed
+      end
+
+      lambda do |event|
+        return event unless event.has_key? field
+        parsers.each do |joda|
+          millis = joda.parseMillis event[field]
+          begin
+            event[target] = Time.at(0.001 * millis).iso8601(precision)
+            return filtered(event, conf)
+          rescue
+          end
+        end
+
+        if error_tag
+          event['tags'] ||= []
+          event['tags'] << error_tag
+        end
+        event
+      end
+    end
+  end
+end

data/lib/anschel/filter.rb

@@ -0,0 +1,41 @@
+require_relative 'filter/convert'
+require_relative 'filter/gsub'
+require_relative 'filter/parse'
+require_relative 'filter/scan'
+require_relative 'filter/stamp'
+
+
+module Anschel
+  class Filter
+
+    attr_reader :filters
+
+    def initialize config
+      @filters = Hash.new { |h,k| h[k] = [] }
+      config.each do |event_type, filter_defns|
+        filter_defns.each do |filter_defn|
+          filter_type = filter_defn.keys.first
+          filter_conf = filter_defn[filter_type]
+          @filters[event_type] << self.send(filter_type, filter_conf)
+        end
+      end
+    end
+
+
+    def apply event
+      raise 'Event does not have a "type" field' unless event['type']
+      filters[:*].each { |f| f.call event }
+      filters[event[:type]].each { |f| f.call event }
+      event
+    end
+
+
+    def filtered event, options
+      if remove_field = options[:remove_field]
+        event.delete remove_field
+      end
+      event
+    end
+
+  end
+end

data/lib/anschel/input.rb

@@ -0,0 +1,21 @@
+require 'jruby-kafka'
+
+
+module Anschel
+  class Input
+    def initialize config
+      qsize  = config.delete(:queue_size) || 1000
+      @queue = SizedQueue.new qsize
+      consumer_group = Kafka::Group.new config
+      consumer_group.run num_cpus, @queue
+
+      trap('SIGINT') do
+        consumer_group.shutdown
+        exit
+      end
+    end
+
+
+    def pop ; @queue.pop end
+  end
+end

data/lib/anschel/main.rb

@@ -0,0 +1,70 @@
+require 'logger'
+require 'tempfile'
+
+require 'jrjackson'
+
+require_relative 'mjolnir'
+require_relative 'metadata'
+
+require_relative 'input'
+require_relative 'filter'
+require_relative 'output'
+
+
+module Anschel
+  class Main < Mjolnir
+
+
+    desc 'version', 'Echo the application version'
+    def version
+      puts VERSION
+    end
+
+
+    desc 'art', 'View the application art'
+    def art
+      puts "\n%s\n" % ART
+    end
+
+
+    desc 'test', 'Test the Anschel agent'
+    option :config, \
+      type: :string,
+      aliases: %w[ -c ],
+      desc: 'Main configuration file',
+      default: '/etc/anschel.json'
+    include_common_options
+    def test
+      config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
+      setup_log4j config[:log4j]
+
+      input  = Input.new config[:kafka]
+      filter = Filter.new config[:filter]
+      output = Output.new config[:elasticsearch]
+
+      start = Time.now
+      count = 0
+
+      ts = num_cpus.times.map do
+        Thread.new do
+          loop do
+            event = JrJackson::Json.load input.pop.message.to_s
+            output.push filter.apply(event)
+            if (count += 1) % 100_000 == 0
+              elapsed = Time.now - start
+              log.info \
+                event: 'stat',
+                count: count,
+                elapsed_s: elapsed.to_f,
+                rate_eps: ( 1.0 * count / elapsed.to_f )
+            end
+          end
+        end
+      end
+
+      ts.map &:join
+      exit
+    end
+
+  end
+end

data/lib/anschel/metadata.rb

@@ -0,0 +1,36 @@
+module Anschel
+  NAME     = 'anschel'
+
+  # A quick summary for use in the command-line interface
+  SUMMARY  = %q.Companion to Franz.
+
+  # Take credit for your work
+  AUTHOR   = 'Sean Clemmer'
+
+  # Take responsibility for your work
+  EMAIL    = 'sczizzo@gmail.com'
+
+  # Like the MIT license, but even simpler
+  LICENSE  = 'ISC'
+
+  # Where you should look first
+  HOMEPAGE = 'https://github.com/sczizzo/anschel'
+
+  # Project root
+  ROOT = File.join File.dirname(__FILE__), '..', '..'
+
+  # Pull the project version out of the VERSION file
+  VERSION = File.read(File.join(ROOT, 'VERSION')).strip
+
+  ART = <<-'EOART'
+                                        888               888
+                                        888               888
+                                        888               888
+     8888b.  88888b.  .d8888b   .d8888b 88888b.   .d88b.  888
+        "88b 888 "88b 88K      d88P"    888 "88b d8P  Y8b 888
+    .d888888 888  888 "Y8888b. 888      888  888 88888888 888
+    888  888 888  888      X88 Y88b.    888  888 Y8b.     888
+    "Y888888 888  888  88888P'  "Y8888P 888  888  "Y8888  888
+
+  EOART
+end

data/lib/anschel/mjolnir.rb

@@ -0,0 +1,44 @@
+require 'thor'
+
+
+module Anschel
+
+  # Thor's hammer! Like Thor with better logging
+  class Mjolnir < Thor
+
+    # Common options for Thor commands
+    COMMON_OPTIONS = {
+      log: {
+        type: :string,
+        aliases: %w[ -L ],
+        desc: 'Log to file instead of STDOUT',
+        default: ENV['ANSCHEL_LOG'] || nil
+      },
+      debug: {
+        type: :boolean,
+        aliases: %w[ -V ],
+        desc: 'Enable DEBUG-level logging',
+        default: ENV['ANSCHEL_DEBUG'] || false
+      }
+    }
+
+    # Decorate Thor commands with the options above
+    def self.include_common_options
+      COMMON_OPTIONS.each do |name, spec|
+        option name, spec
+      end
+    end
+
+
+    no_commands do
+
+      # Construct a Logger given the command-line options
+      def log
+        @logger = Logger.new(options.log || STDOUT)
+        @logger.level = Logger::DEBUG if options.debug?
+        @logger
+      end
+
+    end
+  end
+end

data/lib/anschel/output.rb

@@ -0,0 +1,30 @@
+require 'elasticsearch'
+
+
+module Anschel
+  class Output
+    def initialize config
+      pattern = config.delete(:index_pattern)
+      bsize   = config.delete(:bulk_size) || 500
+      qsize   = config.delete(:queue_size) || 2000
+      @queue  = SizedQueue.new qsize
+      client  = Elasticsearch::Client.new config
+      client.transport.reload_connections!
+
+      Thread.new do
+        loop do
+          events = bsize.times.map { @queue.pop }
+
+          body = events.map do |e|
+            index = e.fetch '@@anschel_index', 'logs-anschel'
+            { index: { _index: index, _type: e['type'], data: e } }
+          end
+
+          client.bulk body: body
+        end
+      end
+    end
+
+    def push event ; @queue << event end
+  end
+end

data/lib/anschel.rb

@@ -0,0 +1,27 @@
+# Return the number of CPU cores (should work on Unixy platforms)
+def num_cpus
+  return Java::Java.lang.Runtime.getRuntime.availableProcessors if defined? Java::Java
+  return File.read('/proc/cpuinfo').scan(/^processor\s*:/).size if File.exist? '/proc/cpuinfo'
+  return `sysctl -a | grep cpu`.split(/\s+/,2).last.to_i
+rescue
+  return 1
+end
+
+def setup_log4j config
+  path = config.delete(:path) || '/var/log/anschel4j.log'
+  pattern = config.delete(:pattern) || '[%d] %p %m (%c)%n'
+  Tempfile.open('anschel_log4j') do |f|
+    log4j = %Q|
+      log4j.rootLogger=INFO, A1
+      log4j.appender.A1=org.apache.log4j.RollingFileAppender
+      log4j.appender.A1.File=%s
+      log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+      log4j.appender.A1.layout.ConversionPattern=%s
+    | % [ path, pattern ]
+    f.write log4j.gsub(/^\s+/,'')
+    f.rewind
+    org.apache.log4j.PropertyConfigurator.configure(f.path)
+  end
+end
+
+require_relative 'anschel/main'

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: anschel
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Sean Clemmer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-06-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: thor
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jruby-kafka
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jrjackson
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Companion to Franz.
+email: sczizzo@gmail.com
+executables:
+- anschel
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- Readme.md
+- VERSION
+- bin/anschel
+- lib/anschel.rb
+- lib/anschel/filter.rb
+- lib/anschel/filter/convert.rb
+- lib/anschel/filter/gsub.rb
+- lib/anschel/filter/parse.rb
+- lib/anschel/filter/scan.rb
+- lib/anschel/filter/stamp.rb
+- lib/anschel/input.rb
+- lib/anschel/main.rb
+- lib/anschel/metadata.rb
+- lib/anschel/mjolnir.rb
+- lib/anschel/output.rb
+homepage: https://github.com/sczizzo/anschel
+licenses:
+- ISC
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.6
+signing_key:
+specification_version: 4
+summary: Companion to Franz
+test_files: []