anschel 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e693593d8211b7f7a7ee60d06536d463f74cb2f
-  data.tar.gz: f67b0087ae0d56c73beed719263feb46e78ec301
+  metadata.gz: b20f566992eecc30983efa61d5a3154da257061b
+  data.tar.gz: 64fcbee6db3ccb50472b4ab545ab56e06279a624
 SHA512:
-  metadata.gz: 0fe919474cdc8d4e31f7ed8703600d960610876613ffca815d79267cf00fdc02e1dfdc5af2e692ae72d0005e67e9372f24da289c4abb1806f09abb06ad8c6f80
-  data.tar.gz: 590ccdd14141ef0feeb4482a8063d634067f9b3ad73d239a2afe0782be6857fa99bc1751a7c2508957e3cf53782c320942eee535899ca6d24a2e69e6ab86ca7f
+  metadata.gz: e83e46bef67b3dc054aef46e7cd5f3e615508b2bf142f770b243393039c410110994a95a239d20b8a4bef51e68cd5aefbef052e1c0459b2a9e15e265052ecdef
+  data.tar.gz: e809c1bb0881fb6717ccacea4b76981debeb53371a82b0aa20043040fec3b632049ccab763de15b8ffebf0830e772eac37e9094df437da9b829a16d00b3a34b9

data/Readme.md

@@ -19,10 +19,10 @@ Just call for help!
 
     $ java -jar anschel-1.2.3.jar help
     Commands:
-      anschel agent           # Run the Anschel agent
-      anschel art             # View the application art
+      anschel agent           # Run application
+      anschel art             # Show application art
       anschel help [COMMAND]  # Describe available commands or one specific command
-      anschel version         # Echo the application version
+      anschel version         # Show application version
 
 Probably you're most interested in the `agent` command:
 
@@ -31,12 +31,15 @@ Probably you're most interested in the `agent` command:
       anschel agent
 
     Options:
-      -c, [--config=CONFIG]        # Main configuration file
+      -c, [--config=CONFIG]        # Path to primary configuration file
                                    # Default: /etc/anschel.json
+      -i, [--stats-interval=N]     # Interval for reporting stats (seconds)
+                                   # Default: 30
       -L, [--log=LOG]              # Log to file instead of STDOUT
       -V, [--debug], [--no-debug]  # Enable DEBUG-level logging
+      -Z, [--trace], [--no-trace]  # Enable TRACE-level logging (precedence over DEBUG)
 
-    Run the Anschel agent
+    Run application
 
 
 
@@ -137,6 +140,8 @@ You might deploy Anschel with Upstart. Here's a minimal config:
 
 ### Changelog
 
-#### v1.0 (devlop)
+#### v1.0
+
+_In devlopment_
 
 - Intial implementation of the Kafka-to-Elasticsearch pipeline

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/anschel/filter/convert.rb

@@ -6,7 +6,8 @@
 # }
 module Anschel
   class Filter
-    def convert conf
+    def convert conf, log
+      log.debug event: 'compile-filter', filter: 'convert', conf: conf
       field = conf.delete :field
       type  = conf.delete :type
 
@@ -21,6 +22,9 @@ module Anschel
         'string'  => :to_s
       }
 
+      log.debug event: 'compiled-filter', filter: 'convert', \
+        field: field, type: type
+
       lambda do |event|
         return event unless event.has_key? field
         event[field] = event[field].send type_conversions[type]

data/lib/anschel/filter/gsub.rb

@@ -7,7 +7,7 @@
 # }
 module Anschel
   class Filter
-    def gsub conf
+    def gsub conf, log
       field   = conf.delete :field
       match   = Regexp.new conf.delete(:match)
       replace = conf.delete :replace
@@ -18,6 +18,10 @@ module Anschel
 
       field = field.to_sym
 
+
+      log.debug event: 'compiled-filter', filter: 'gsub', \
+        field: field, match: match, replace: replace
+
       lambda do |event|
         return event unless event.has_key? field
         event[field].gsub! match, replace

data/lib/anschel/filter/index.rb

@@ -1,6 +1,13 @@
+# {
+#   "index": {
+#     "stamp": "",
+#     "prefix": "",
+#     "suffix": ""
+#   }
+# }
 module Anschel
   class Filter
-    def index conf
+    def index conf, log
       stamp  = conf.delete(:stamp)  || '@timestamp'
       prefix = conf.delete(:prefix) || 'logs-%{type}-'
       suffix = conf.delete(:suffix) || '%Y.%m.%d'
@@ -13,6 +20,10 @@ module Anschel
       joda = joda.withDefaultYear(Time.new.year)
       joda = joda.withOffsetParsed
 
+
+      log.debug event: 'compiled-filter', filter: 'index', \
+        stamp: stamp, prefix: prefix, suffix: suffix
+
       lambda do |event|
         return event unless event.has_key? stamp
         millis     = joda.parseMillis event[stamp]

data/lib/anschel/filter/parse.rb

@@ -6,7 +6,7 @@
 # }
 module Anschel
   class Filter
-    def parse conf
+    def parse conf, log
       field   = conf.delete :field
       pattern = Regexp.new conf.delete(:pattern)
 
@@ -15,6 +15,10 @@ module Anschel
 
       field = field.to_sym
 
+
+      log.debug event: 'compiled-filter', filter: 'parse', \
+        field: field, pattern: pattern
+
       lambda do |event|
         return event unless event.has_key? field
         mdata = pattern.match event[field]

data/lib/anschel/filter/scan.rb

@@ -7,7 +7,7 @@
 # }
 module Anschel
   class Filter
-    def scan conf
+    def scan conf, log
       field   = conf.delete :field
       pattern = Regexp.new conf.delete(:pattern)
       target  = conf.delete :target
@@ -19,6 +19,10 @@ module Anschel
       field  = field.to_sym
       target = target.to_sym
 
+
+      log.debug event: 'compiled-filter', filter: 'scan', \
+        field: field, pattern: pattern, target: target
+
       lambda do |event|
         return event unless event.has_key? field
         results = event[field].scan(pattern).flatten.uniq

data/lib/anschel/filter/stamp.rb

@@ -6,19 +6,17 @@
 # }
 module Anschel
   class Filter
-    def stamp conf
+    def stamp conf, log
       field     = conf.delete :field
       pattern   = conf.delete :pattern
       target    = conf.delete :target
-      precision = conf.delete :precision
       error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'stamp-error'
 
       raise 'Missing required "field" for "stamp" filter' if field.nil?
       raise 'Missing required "pattern" for "stamp" filter' if pattern.nil?
 
-      patterns  = pattern.is_a?(Array) ? pattern : [ pattern ]
-      precision = (precision || 3).to_i
-      target  ||= '@timestamp'
+      patterns = pattern.is_a?(Array) ? pattern : [ pattern ]
+      target ||= '@timestamp'
 
       field  = field.to_sym
       target = target.to_sym
@@ -29,12 +27,16 @@ module Anschel
         joda = joda.withOffsetParsed
       end
 
+
+      log.debug event: 'compiled-filter', filter: 'stamp', \
+        field: field, pattern: pattern, target: target, error_tag: error_tag
+
       lambda do |event|
         return event unless event.has_key? field
         parsers.each do |joda|
           millis = joda.parseMillis event[field]
           begin
-            event[target] = Time.at(0.001 * millis).iso8601(precision)
+            event[target] = Time.at(0.001 * millis).iso8601(3)
             return filtered(event, conf)
           rescue
           end

data/lib/anschel/filter.rb

@@ -11,13 +11,14 @@ module Anschel
 
     attr_reader :filters
 
-    def initialize config, log
+    def initialize config, stats, log
+      log.trace event: 'filter', config: config
       @filters = Hash.new { |h,k| h[k] = [] }
       config.each do |event_type, filter_defns|
         filter_defns.each do |filter_defn|
           filter_type = filter_defn.keys.first
           filter_conf = filter_defn[filter_type]
-          @filters[event_type] << self.send(filter_type, filter_conf)
+          @filters[event_type] << self.send(filter_type, filter_conf, log)
         end
       end
       log.info event: 'filter-loaded'

data/lib/anschel/input.rb

@@ -3,7 +3,8 @@ require 'jruby-kafka'
 
 module Anschel
   class Input
-    def initialize config, log
+    def initialize config, stats, log
+      log.trace event: 'input', config: config
       qsize  = config.delete(:queue_size) || 1000
       @queue = SizedQueue.new qsize
       consumer_group = Kafka::Group.new config
@@ -11,7 +12,7 @@ module Anschel
 
       trap('SIGINT') do
         consumer_group.shutdown
-        log.info event: 'goodbye'
+        log.info event: 'goodbye', version: VERSION
         exit
       end
 

data/lib/anschel/main.rb

@@ -5,7 +5,7 @@ require 'jrjackson'
 
 require_relative 'mjolnir'
 require_relative 'metadata'
-
+require_relative 'stats'
 require_relative 'input'
 require_relative 'filter'
 require_relative 'output'
@@ -15,37 +15,42 @@ module Anschel
   class Main < Mjolnir
 
 
-    desc 'version', 'Echo the application version'
+    desc 'version', 'Show application version'
     def version
       puts VERSION
     end
 
 
-    desc 'art', 'View the application art'
+    desc 'art', 'Show application art'
     def art
       puts "\n%s\n" % ART
     end
 
 
-    desc 'agent', 'Run the Anschel agent'
+    desc 'agent', 'Run application'
     option :config, \
       type: :string,
       aliases: %w[ -c ],
-      desc: 'Main configuration file',
+      desc: 'Path to primary configuration file',
       default: '/etc/anschel.json'
+    option :stats_interval, \
+      type: :numeric,
+      aliases: %w[ -i ],
+      desc: 'Interval for reporting stats (seconds)',
+      default: 30
     include_common_options
     def agent
-      log.info event: 'hello'
+      log.info event: 'hello', version: VERSION, options: options.to_hash
       config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
       setup_log4j config[:log4j]
 
-      input  = Input.new config[:kafka], log
-      filter = Filter.new config[:filter], log
-      output = Output.new config[:elasticsearch], log
+      stats  = Stats.new log, options.stats_interval
+      input  = Input.new config[:kafka], stats, log
+      filter = Filter.new config[:filter], stats, log
+      output = Output.new config[:elasticsearch], stats, log
 
-      start  = Time.now
-      count  = 0
-      sample = 100_000
+      stats.create 'count', 0
+      stats.get 'count'
 
       ts = num_cpus.times.map do
         Thread.new do
@@ -53,17 +58,7 @@ module Anschel
             event = JrJackson::Json.load \
               input.shift.message.to_s, symbolize_keys: true
             output.push filter.apply(event)
-            if (count += 1) % sample == 0
-              old_count, now = count, Time.now
-              elapsed, start = (now - start).to_f, now
-              rate           = 1.0 * sample / elapsed
-              log.info \
-                event: 'stat',
-                count: old_count,
-                sample: sample,
-                elapsed_s: elapsed,
-                rate_eps: rate
-            end
+            stats.inc 'count'
           end
         end
       end

data/lib/anschel/metadata.rb

@@ -21,15 +21,16 @@ module Anschel
   # Pull the project version out of the VERSION file
   VERSION = File.read(File.join(ROOT, 'VERSION')).strip
 
+  # Thank you, http://patorjk.com/software/taag
   ART = <<-'EOART'
-                                        888               888
-                                        888               888
-                                        888               888
-     8888b.  88888b.  .d8888b   .d8888b 88888b.   .d88b.  888
-        "88b 888 "88b 88K      d88P"    888 "88b d8P  Y8b 888
-    .d888888 888  888 "Y8888b. 888      888  888 88888888 888
-    888  888 888  888      X88 Y88b.    888  888 Y8b.     888
-    "Y888888 888  888  88888P'  "Y8888P 888  888  "Y8888  888
+
+                                             oooo                  oooo
+                                             `888                  `888
+     .oooo.   ooo. .oo.    .oooo.o  .ooooo.   888 .oo.    .ooooo.   888
+    `P  )88b  `888P"Y88b  d88(  "8 d88' `"Y8  888P"Y88b  d88' `88b  888
+     .oP"888   888   888  `"Y88b.  888        888   888  888ooo888  888
+    d8(  888   888   888  o.  )88b 888   .o8  888   888  888    .o  888
+    `Y888""8o o888o o888o 8""888P' `Y8bod8P' o888o o888o `Y8bod8P' o888o
 
   EOART
 end

data/lib/anschel/mjolnir.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'slog'
 
 
 module Anschel
@@ -19,6 +20,12 @@ module Anschel
         aliases: %w[ -V ],
         desc: 'Enable DEBUG-level logging',
         default: ENV['ANSCHEL_DEBUG'] || false
+      },
+      trace: {
+        type: :boolean,
+        aliases: %w[ -Z ],
+        desc: 'Enable TRACE-level logging (precedence over DEBUG)',
+        default: ENV['ANSCHEL_TRACE'] || false
       }
     }
 
@@ -35,8 +42,9 @@ module Anschel
       # Construct a Logger given the command-line options
       def log
         return @logger if defined? @logger
-        @logger = Logger.new(options.log || STDOUT)
-        @logger.level = Logger::DEBUG if options.debug?
+        @logger = Slog.new out: (options.log || $stdout), prettify: false
+        @logger.level = :debug if options.debug?
+        @logger.level = :trace if options.trace?
         @logger
       end
 

data/lib/anschel/output.rb

@@ -7,11 +7,12 @@ require 'elasticsearch'
 
 module Anschel
   class Output
-    def initialize config, log
+    def initialize config, stats, log
+      log.trace event: 'output', config: config
       pattern = config.delete(:index_pattern)
       qsize   = config.delete(:queue_size) || 2000
       bsize   = config.delete(:bulk_size) || 500
-      timeout = config.delete(:bulk_timeout) || 1.0
+      timeout = config.delete(:bulk_timeout) || 0.5
       slice   = timeout / bsize
       client  = Elasticsearch::Client.new config
       client.transport.reload_connections!

data/lib/anschel/stats.rb

@@ -0,0 +1,88 @@
+require 'thread'
+require 'logger'
+
+
+module Anschel
+  class Stats
+
+    def initialize logger, interval=60
+      @logger   = logger
+      @interval = interval
+      @stats    = Hash.new
+      @lock     = Mutex.new
+
+      Thread.new do
+        loop do
+          sleep @interval
+          report_and_reset
+        end
+      end
+
+      @logger.info event: 'stats-loaded'
+    end
+
+
+    def create name, default=nil
+      with_lock do
+        stats[name] = Hash.new { |h,k| h[k] = default }
+      end
+    end
+
+    def delete name
+      with_lock do
+        stats.delete name
+      end
+    end
+
+    def inc name, by=1
+      with_lock do
+        stats[name][:val] += by
+      end
+    end
+
+    def dec name, by=1
+      with_lock do
+        stats[name][:val] -= by
+      end
+    end
+
+    def set name, to
+      with_lock do
+        stats[name][:val] = to
+      end
+    end
+
+    def get name
+      with_lock do
+        stats[name][:val]
+      end
+    end
+
+
+  private
+    attr_reader :stats
+
+    def with_lock &block
+      @lock.synchronize do
+        yield
+      end
+    end
+
+    def report_and_reset
+      ready_stats = with_lock do
+        stats.keys.flat_map do |k|
+          v = stats[k].delete(:val)
+          [
+            [ k, v ],
+            [ "#{k}_rate", 1.0 * v / @interval ]
+          ]
+        end
+      end
+
+      @logger.info \
+        event: 'stats',
+        interval: @interval,
+        stats: Hash[ready_stats]
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sean Clemmer
@@ -80,6 +80,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: slog
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Logstash-like for moving events from Kafka into Elasticsearch.
 email: sczizzo@gmail.com
 executables:
@@ -104,6 +118,7 @@ files:
 - lib/anschel/metadata.rb
 - lib/anschel/mjolnir.rb
 - lib/anschel/output.rb
+- lib/anschel/stats.rb
 homepage: https://github.com/sczizzo/anschel
 licenses:
 - ISC