anschel 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c9d437249846bfe1e7ab107fb86e8338a709d8d
-  data.tar.gz: 75e4124f0cd424865aa63692d2fdb16001e9bb58
+  metadata.gz: 4e693593d8211b7f7a7ee60d06536d463f74cb2f
+  data.tar.gz: f67b0087ae0d56c73beed719263feb46e78ec301
 SHA512:
-  metadata.gz: 61fd41165f874a2aa55677b3d3a562994e9283d67c949a2d8945df295e1a9d9554af508ce2c39e05b781b03dc8f8f562de1ef3d4c5af8da23c67084ef375bccd
-  data.tar.gz: 4bb4b827b537644b7f5495a42dccf79d867e8f0a76c8cfc17c5abd2df1b91758c8f1271d8147854a2a56871ba2d34c76afa0940ec7b6d6834fca6fafb9110962
+  metadata.gz: 0fe919474cdc8d4e31f7ed8703600d960610876613ffca815d79267cf00fdc02e1dfdc5af2e692ae72d0005e67e9372f24da289c4abb1806f09abb06ad8c6f80
+  data.tar.gz: 590ccdd14141ef0feeb4482a8063d634067f9b3ad73d239a2afe0782be6857fa99bc1751a7c2508957e3cf53782c320942eee535899ca6d24a2e69e6ab86ca7f

data/Readme.md

@@ -1,3 +1,142 @@
-# Anschel
+# Anschel ![Version](https://img.shields.io/gem/v/anschel.svg?style=flat-square)
 
-Companion to Franz
+Logstash-like for moving events from Kafka into Elasticsearch.
+
+
+
+## Usage, Configuration &c.
+
+### Installation
+
+Download the jarfile from the [GitHub releases page](https://github.com/sczizzo/anschel/releases)
+and run like so:
+
+    $ java -jar anschel-1.2.3.jar
+
+### Usage
+
+Just call for help!
+
+    $ java -jar anschel-1.2.3.jar help
+    Commands:
+      anschel agent           # Run the Anschel agent
+      anschel art             # View the application art
+      anschel help [COMMAND]  # Describe available commands or one specific command
+      anschel version         # Echo the application version
+
+Probably you're most interested in the `agent` command:
+
+    $ java -jar anschel-1.2.3.jar help agent
+    Usage:
+      anschel agent
+
+    Options:
+      -c, [--config=CONFIG]        # Main configuration file
+                                   # Default: /etc/anschel.json
+      -L, [--log=LOG]              # Log to file instead of STDOUT
+      -V, [--debug], [--no-debug]  # Enable DEBUG-level logging
+
+    Run the Anschel agent
+
+
+
+### Configuration
+
+It's kinda like a JSON version of the Logstash config language:
+
+    {
+      "log4j": {
+        "path": "/path/to/anschel4j.log",
+        "pattern": "[%d] %p %m (%c)%n"
+      },
+      "kafka": {
+        "queue_size": 2000,
+        "zk_connect": "localhost:2181",
+        "zk_connect_timeout": 6000,
+        "zk_session_timeout": 6000,
+        "group_id": "anschel",
+        "topic_id": "franz",
+        "reset_beginning": null,
+        "auto_offset_reset": "smallest",
+        "consumer_restart_on_error": true,
+        "auto_commit_interval": 1000,
+        "rebalance_max_retries": 4,
+        "rebalance_backoff_ms": 2000,
+        "socket_timeout_ms": 30000,
+        "socket_receive_buffer_bytes": 65536,
+        "fetch_message_max_bytes": 1048576,
+        "auto_commit_enable": true,
+        "queued_max_message_chunks": 10,
+        "fetch_min_bytes": 1,
+        "fetch_wait_max_ms": 100,
+        "refresh_leader_backoff_ms": 200,
+        "consumer_timeout_ms": -1,
+        "consumer_restart_sleep_ms": 0
+      },
+      "elasticsearch": {
+        "queue_size": 2000,
+        "bulk_size": 200,
+        "hosts": [ "localhost:9200" ],
+        "randomize_hosts": true,
+        "reload_connections": true,
+        "reload_on_failure": true,
+        "sniffer_timeout": 5
+      },
+      "filter": {
+        "type": [
+          {
+            "scan": {
+              "field": "message",
+              "pattern": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}",
+              "target": "guids"
+            }
+          },
+          {
+            "scan": {
+              "field": "path",
+              "pattern": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}",
+              "target": "guids"
+            }
+          }
+        ],
+        "_after": [
+          {
+            "gsub": {
+              "field": "type",
+              "match": "-.*",
+              "replace": ""
+            }
+          },
+          {
+            "index": {}
+          }
+        ]
+      }
+    }
+
+
+
+
+### Operation
+
+You might deploy Anschel with Upstart. Here's a minimal config:
+
+    #!upstart
+    description "anschel"
+
+    console log
+
+    start on startup
+    stop on shutdown
+    respawn
+
+    exec java -jar anschel-1.2.3.jar \
+      --config /etc/anschel.json --log /var/log/anschel.log
+
+
+
+### Changelog
+
+#### v1.0 (devlop)
+
+- Intial implementation of the Kafka-to-Elasticsearch pipeline

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.1.0

data/lib/anschel/filter/convert.rb

@@ -13,6 +13,8 @@ module Anschel
       raise 'Missing required "field" for "convert" filter' if field.nil?
       raise 'Missing required "type" for "convert" filter' if type.nil?
 
+      field = field.to_sym
+
       type_conversions = {
         'integer' => :to_i,
         'float'   => :to_f,

data/lib/anschel/filter/gsub.rb

@@ -16,6 +16,8 @@ module Anschel
       raise 'Missing required "match" for "gsub" filter' if match.nil?
       raise 'Missing required "replace" for "gsub" filter' if replace.nil?
 
+      field = field.to_sym
+
       lambda do |event|
         return event unless event.has_key? field
         event[field].gsub! match, replace

data/lib/anschel/filter/index.rb

@@ -0,0 +1,26 @@
+module Anschel
+  class Filter
+    def index conf
+      stamp  = conf.delete(:stamp)  || '@timestamp'
+      prefix = conf.delete(:prefix) || 'logs-%{type}-'
+      suffix = conf.delete(:suffix) || '%Y.%m.%d'
+
+      stamp = stamp.to_sym
+
+      iso8601 = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
+
+      joda = org.joda.time.format.DateTimeFormat.forPattern iso8601
+      joda = joda.withDefaultYear(Time.new.year)
+      joda = joda.withOffsetParsed
+
+      lambda do |event|
+        return event unless event.has_key? stamp
+        millis     = joda.parseMillis event[stamp]
+        idx_prefix = prefix % event
+        idx_suffix = Time.at(0.001 * millis).strftime(suffix)
+        event[:_index] = idx_prefix + idx_suffix
+        filtered event, conf
+      end
+    end
+  end
+end

data/lib/anschel/filter/parse.rb

@@ -13,6 +13,8 @@ module Anschel
       raise 'Missing required "field" for "parse" filter' if field.nil?
       raise 'Missing required "pattern" for "parse" filter' if pattern.nil?
 
+      field = field.to_sym
+
       lambda do |event|
         return event unless event.has_key? field
         mdata = pattern.match event[field]

data/lib/anschel/filter/scan.rb

@@ -16,6 +16,9 @@ module Anschel
       raise 'Missing required "pattern" for "scan" filter' if pattern.nil?
       raise 'Missing required "target" for "convert" filter' if target.nil?
 
+      field  = field.to_sym
+      target = target.to_sym
+
       lambda do |event|
         return event unless event.has_key? field
         results = event[field].scan(pattern).flatten.uniq

data/lib/anschel/filter/stamp.rb

@@ -20,9 +20,12 @@ module Anschel
       precision = (precision || 3).to_i
       target  ||= '@timestamp'
 
+      field  = field.to_sym
+      target = target.to_sym
+
       parsers = patterns.map do |p|
-        joda = org.joda.time.format.DateTimeFormat.forPattern(p)
-        joda = joda.withDefaultYear(Time.new.year)
+        joda = org.joda.time.format.DateTimeFormat.forPattern p
+        joda = joda.withDefaultYear Time.new.year
         joda = joda.withOffsetParsed
       end
 
@@ -38,8 +41,8 @@ module Anschel
         end
 
         if error_tag
-          event['tags'] ||= []
-          event['tags'] << error_tag
+          event[:tags] ||= []
+          event[:tags] << error_tag
         end
         event
       end

data/lib/anschel/filter.rb

@@ -1,5 +1,6 @@
 require_relative 'filter/convert'
 require_relative 'filter/gsub'
+require_relative 'filter/index'
 require_relative 'filter/parse'
 require_relative 'filter/scan'
 require_relative 'filter/stamp'
@@ -10,7 +11,7 @@ module Anschel
 
     attr_reader :filters
 
-    def initialize config
+    def initialize config, log
       @filters = Hash.new { |h,k| h[k] = [] }
       config.each do |event_type, filter_defns|
         filter_defns.each do |filter_defn|
@@ -19,13 +20,16 @@ module Anschel
           @filters[event_type] << self.send(filter_type, filter_conf)
         end
       end
+      log.info event: 'filter-loaded'
     end
 
 
     def apply event
-      raise 'Event does not have a "type" field' unless event['type']
-      filters[:*].each { |f| f.call event }
-      filters[event[:type]].each { |f| f.call event }
+      raise 'Event does not have a "type" field' unless event[:type]
+      type = event[:type].dup # In case of modification
+      filters[:_before].each { |f| f.call event }
+      filters[type].each     { |f| f.call event }
+      filters[:_after].each  { |f| f.call event }
       event
     end
 

data/lib/anschel/input.rb

@@ -3,7 +3,7 @@ require 'jruby-kafka'
 
 module Anschel
   class Input
-    def initialize config
+    def initialize config, log
       qsize  = config.delete(:queue_size) || 1000
       @queue = SizedQueue.new qsize
       consumer_group = Kafka::Group.new config
@@ -11,11 +11,14 @@ module Anschel
 
       trap('SIGINT') do
         consumer_group.shutdown
+        log.info event: 'goodbye'
         exit
       end
+
+      log.info event: 'input-loaded'
     end
 
 
-    def pop ; @queue.pop end
+    def shift ; @queue.shift end
   end
 end

data/lib/anschel/main.rb

@@ -27,41 +27,48 @@ module Anschel
     end
 
 
-    desc 'test', 'Test the Anschel agent'
+    desc 'agent', 'Run the Anschel agent'
     option :config, \
       type: :string,
       aliases: %w[ -c ],
       desc: 'Main configuration file',
       default: '/etc/anschel.json'
     include_common_options
-    def test
+    def agent
+      log.info event: 'hello'
       config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
       setup_log4j config[:log4j]
 
-      input  = Input.new config[:kafka]
-      filter = Filter.new config[:filter]
-      output = Output.new config[:elasticsearch]
+      input  = Input.new config[:kafka], log
+      filter = Filter.new config[:filter], log
+      output = Output.new config[:elasticsearch], log
 
-      start = Time.now
-      count = 0
+      start  = Time.now
+      count  = 0
+      sample = 100_000
 
       ts = num_cpus.times.map do
         Thread.new do
           loop do
-            event = JrJackson::Json.load input.pop.message.to_s
+            event = JrJackson::Json.load \
+              input.shift.message.to_s, symbolize_keys: true
             output.push filter.apply(event)
-            if (count += 1) % 100_000 == 0
-              elapsed = Time.now - start
+            if (count += 1) % sample == 0
+              old_count, now = count, Time.now
+              elapsed, start = (now - start).to_f, now
+              rate           = 1.0 * sample / elapsed
               log.info \
                 event: 'stat',
-                count: count,
-                elapsed_s: elapsed.to_f,
-                rate_eps: ( 1.0 * count / elapsed.to_f )
+                count: old_count,
+                sample: sample,
+                elapsed_s: elapsed,
+                rate_eps: rate
             end
           end
         end
       end
 
+      log.info event: 'fully-loaded'
       ts.map &:join
       exit
     end

data/lib/anschel/metadata.rb

@@ -1,8 +1,7 @@
 module Anschel
+  # In a nutshell
   NAME     = 'anschel'
-
-  # A quick summary for use in the command-line interface
-  SUMMARY  = %q.Companion to Franz.
+  SUMMARY  = %q.Logstash-like for moving events from Kafka into Elasticsearch.
 
   # Take credit for your work
   AUTHOR   = 'Sean Clemmer'

data/lib/anschel/mjolnir.rb

@@ -34,6 +34,7 @@ module Anschel
 
       # Construct a Logger given the command-line options
       def log
+        return @logger if defined? @logger
         @logger = Logger.new(options.log || STDOUT)
         @logger.level = Logger::DEBUG if options.debug?
         @logger

data/lib/anschel/output.rb

@@ -1,30 +1,50 @@
+require 'thread'
+
+require 'typhoeus/adapters/faraday'
+require 'typhoeus'
 require 'elasticsearch'
 
 
 module Anschel
   class Output
-    def initialize config
+    def initialize config, log
       pattern = config.delete(:index_pattern)
-      bsize   = config.delete(:bulk_size) || 500
       qsize   = config.delete(:queue_size) || 2000
-      @queue  = SizedQueue.new qsize
+      bsize   = config.delete(:bulk_size) || 500
+      timeout = config.delete(:bulk_timeout) || 1.0
+      slice   = timeout / bsize
       client  = Elasticsearch::Client.new config
       client.transport.reload_connections!
 
+      @queue = SizedQueue.new qsize
+
       Thread.new do
         loop do
-          events = bsize.times.map { @queue.pop }
+          events = []
+          count  = 0
+          start  = Time.now.to_f
+          until (Time.now.to_f - start > timeout) || ((count += 1) > bsize)
+            begin
+              events.push @queue.shift(true)
+            rescue # shift returned immediately
+              sleep slice
+            end
+          end
+
+          next if events.empty?
 
           body = events.map do |e|
-            index = e.fetch '@@anschel_index', 'logs-anschel'
-            { index: { _index: index, _type: e['type'], data: e } }
+            index = e.delete(:_index) || 'logs-anschel'
+            { index: { _index: index, _type: e[:type], data: e } }
           end
 
           client.bulk body: body
         end
       end
+
+      log.info event: 'output-loaded'
     end
 
-    def push event ; @queue << event end
+    def push event ; @queue.push event end
   end
 end

data/lib/anschel.rb

@@ -24,4 +24,6 @@ def setup_log4j config
   end
 end
 
+Thread.abort_on_exception = true
+
 require_relative 'anschel/main'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-21 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,21 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Companion to Franz.
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: typhoeus
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Logstash-like for moving events from Kafka into Elasticsearch.
 email: sczizzo@gmail.com
 executables:
 - anschel
@@ -81,6 +95,7 @@ files:
 - lib/anschel/filter.rb
 - lib/anschel/filter/convert.rb
 - lib/anschel/filter/gsub.rb
+- lib/anschel/filter/index.rb
 - lib/anschel/filter/parse.rb
 - lib/anschel/filter/scan.rb
 - lib/anschel/filter/stamp.rb
@@ -112,5 +127,5 @@ rubyforge_project:
 rubygems_version: 2.4.6
 signing_key:
 specification_version: 4
-summary: Companion to Franz
+summary: Logstash-like for moving events from Kafka into Elasticsearch
 test_files: []