anschel 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b20f566992eecc30983efa61d5a3154da257061b
-  data.tar.gz: 64fcbee6db3ccb50472b4ab545ab56e06279a624
+  metadata.gz: 6effb5e9164bcf730b5bab24484ad5af1259021b
+  data.tar.gz: 1ba0f0a949a64850aa81eb80ed6852eb896fb824
 SHA512:
-  metadata.gz: e83e46bef67b3dc054aef46e7cd5f3e615508b2bf142f770b243393039c410110994a95a239d20b8a4bef51e68cd5aefbef052e1c0459b2a9e15e265052ecdef
-  data.tar.gz: e809c1bb0881fb6717ccacea4b76981debeb53371a82b0aa20043040fec3b632049ccab763de15b8ffebf0830e772eac37e9094df437da9b829a16d00b3a34b9
+  metadata.gz: 461945890b911f2d2d1f4a341c0d76d6d38eb484614f01ed4195d7cbb173f1309d95024a06d7f03e1709411418a6d9a50b0556850ae6e0e5c9676a39f77a0298
+  data.tar.gz: 541b50a3e3cb82fed6d195b5efbaf2d366bfb8c74bf5f5240eaad548ff09d5a4b4028be56b354d0ff63f9152f232f46cf6546e311f6402dc6b4159143e10a623

data/Readme.md

@@ -11,13 +11,13 @@ Logstash-like for moving events from Kafka into Elasticsearch.
 Download the jarfile from the [GitHub releases page](https://github.com/sczizzo/anschel/releases)
 and run like so:
 
-    $ java -jar anschel-1.2.3.jar
+    $ java -jar anschel-x.y.z.jar
 
 ### Usage
 
 Just call for help!
 
-    $ java -jar anschel-1.2.3.jar help
+    $ java -jar anschel-x.y.z.jar help
     Commands:
       anschel agent           # Run application
       anschel art             # Show application art
@@ -26,7 +26,7 @@ Just call for help!
 
 Probably you're most interested in the `agent` command:
 
-    $ java -jar anschel-1.2.3.jar help agent
+    $ java -jar anschel-x.y.z.jar help agent
     Usage:
       anschel agent
 
@@ -35,9 +35,9 @@ Probably you're most interested in the `agent` command:
                                    # Default: /etc/anschel.json
       -i, [--stats-interval=N]     # Interval for reporting stats (seconds)
                                    # Default: 30
-      -L, [--log=LOG]              # Log to file instead of STDOUT
-      -V, [--debug], [--no-debug]  # Enable DEBUG-level logging
-      -Z, [--trace], [--no-trace]  # Enable TRACE-level logging (precedence over DEBUG)
+      -l, [--log=LOG]              # Log to file instead of STDOUT
+      -v, [--debug], [--no-debug]  # Enable DEBUG-level logging
+      -z, [--trace], [--no-trace]  # Enable TRACE-level logging (overrides DEBUG)
 
     Run application
 
@@ -48,10 +48,14 @@ Probably you're most interested in the `agent` command:
 It's kinda like a JSON version of the Logstash config language:
 
     {
+      // Anschel requires JRuby, and some libraries log via Log4j
       "log4j": {
         "path": "/path/to/anschel4j.log",
         "pattern": "[%d] %p %m (%c)%n"
       },
+
+      // Kafka is the only supported input; see the `jruby-kafka` homepage for
+      // more details: https://github.com/joekiller/jruby-kafka
       "kafka": {
         "queue_size": 2000,
         "zk_connect": "localhost:2181",
@@ -76,6 +80,9 @@ It's kinda like a JSON version of the Logstash config language:
         "consumer_timeout_ms": -1,
         "consumer_restart_sleep_ms": 0
       },
+
+      // There's only one output, Elasticsearch; see the `elasticsearch-ruby`
+      // homepage for more: https://github.com/elastic/elasticsearch-ruby
       "elasticsearch": {
         "queue_size": 2000,
         "bulk_size": 200,
@@ -85,31 +92,37 @@ It's kinda like a JSON version of the Logstash config language:
         "reload_on_failure": true,
         "sniffer_timeout": 5
       },
+
+      // Just like Logstash, Anschel has a notion of filters
       "filter": {
-        "type": [
+
+        // "_before" is a special set of filters executed on every event before
+        // the main type-specific filters are applied
+        "_before": [
           {
-            "scan": {
-              "field": "message",
-              "pattern": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}",
-              "target": "guids"
+            "gsub": {
+              "field": "type",
+              "match": "-.*",
+              "replace": ""
             }
-          },
+          }
+        ],
+
+        // Each type is allowed any number of filters; see `lib/anschel/filter`
+        // for configuration options and available filters
+        "some-type": [
           {
             "scan": {
-              "field": "path",
+              "field": "message",
               "pattern": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}",
               "target": "guids"
             }
           }
         ],
+
+        // Any "_after" filters will be applied to all events, like "_before"
+        // (but, you know, like after the main filters)
         "_after": [
-          {
-            "gsub": {
-              "field": "type",
-              "match": "-.*",
-              "replace": ""
-            }
-          },
           {
             "index": {}
           }
@@ -133,14 +146,14 @@ You might deploy Anschel with Upstart. Here's a minimal config:
     stop on shutdown
     respawn
 
-    exec java -jar anschel-1.2.3.jar \
+    exec java -jar anschel-x.y.z.jar \
       --config /etc/anschel.json --log /var/log/anschel.log
 
 
 
-### Changelog
+## Changelog
 
-#### v1.0
+### v1.0
 
 _In devlopment_
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/lib/anschel/filter.rb

@@ -13,6 +13,7 @@ module Anschel
 
     def initialize config, stats, log
       log.trace event: 'filter', config: config
+      config ||= {} # Allow for nil config
       @filters = Hash.new { |h,k| h[k] = [] }
       config.each do |event_type, filter_defns|
         filter_defns.each do |filter_defn|
@@ -35,6 +36,8 @@ module Anschel
     end
 
 
+  private
+
     def filtered event, options
       if remove_field = options[:remove_field]
         event.delete remove_field

data/lib/anschel/main.rb

@@ -40,7 +40,12 @@ module Anschel
       default: 30
     include_common_options
     def agent
-      log.info event: 'hello', version: VERSION, options: options.to_hash
+      log.info \
+        event: 'hello',
+        version: VERSION,
+        options: options.to_hash,
+        num_cpus: num_cpus
+
       config = JrJackson::Json.load File.read(options.config), symbolize_keys: true
       setup_log4j config[:log4j]
 
@@ -49,8 +54,8 @@ module Anschel
       filter = Filter.new config[:filter], stats, log
       output = Output.new config[:elasticsearch], stats, log
 
-      stats.create 'count', 0
-      stats.get 'count'
+      stats.create 'event'
+      stats.get 'event'
 
       ts = num_cpus.times.map do
         Thread.new do
@@ -58,7 +63,7 @@ module Anschel
             event = JrJackson::Json.load \
               input.shift.message.to_s, symbolize_keys: true
             output.push filter.apply(event)
-            stats.inc 'count'
+            stats.inc 'event'
           end
         end
       end

data/lib/anschel/mjolnir.rb

@@ -11,20 +11,20 @@ module Anschel
     COMMON_OPTIONS = {
       log: {
         type: :string,
-        aliases: %w[ -L ],
+        aliases: %w[ -l ],
         desc: 'Log to file instead of STDOUT',
         default: ENV['ANSCHEL_LOG'] || nil
       },
       debug: {
         type: :boolean,
-        aliases: %w[ -V ],
+        aliases: %w[ -v ],
         desc: 'Enable DEBUG-level logging',
         default: ENV['ANSCHEL_DEBUG'] || false
       },
       trace: {
         type: :boolean,
-        aliases: %w[ -Z ],
-        desc: 'Enable TRACE-level logging (precedence over DEBUG)',
+        aliases: %w[ -z ],
+        desc: 'Enable TRACE-level logging (overrides DEBUG)',
         default: ENV['ANSCHEL_TRACE'] || false
       }
     }
@@ -48,6 +48,25 @@ module Anschel
         @logger
       end
 
+      # Write out a temporary properties file for Log4j
+      def setup_log4j config
+        config ||= {}
+        path     = config.delete(:path)    || '/dev/stdout'
+        pattern  = config.delete(:pattern) || '%p [%d] (%c) %m%n'
+        Tempfile.open('anschel_log4j') do |f|
+          log4j = %Q|
+            log4j.rootLogger=INFO, A1
+            log4j.appender.A1=org.apache.log4j.RollingFileAppender
+            log4j.appender.A1.File=%s
+            log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+            log4j.appender.A1.layout.ConversionPattern=%s
+          | % [ path, pattern ]
+          f.write log4j.gsub(/^\s+/,'')
+          f.rewind
+          org.apache.log4j.PropertyConfigurator.configure(f.path)
+        end
+      end
+
     end
   end
 end

data/lib/anschel/stats.rb

@@ -22,9 +22,11 @@ module Anschel
     end
 
 
-    def create name, default=nil
+    def create name, default=0
       with_lock do
-        stats[name] = Hash.new { |h,k| h[k] = default }
+        stats[name] = {}
+        stats[name].default = default
+        stats
       end
     end
 
@@ -72,6 +74,8 @@ module Anschel
       ready_stats = with_lock do
         stats.keys.flat_map do |k|
           v = stats[k].delete(:val)
+          v ||= stats[k].default
+          stats[k][:val] = stats[k].default
           [
             [ k, v ],
             [ "#{k}_rate", 1.0 * v / @interval ]

data/lib/anschel.rb

@@ -1,29 +1,14 @@
+require 'thread'
+
+Thread.abort_on_exception = true
+
 # Return the number of CPU cores (should work on Unixy platforms)
 def num_cpus
   return Java::Java.lang.Runtime.getRuntime.availableProcessors if defined? Java::Java
   return File.read('/proc/cpuinfo').scan(/^processor\s*:/).size if File.exist? '/proc/cpuinfo'
   return `sysctl -a | grep cpu`.split(/\s+/,2).last.to_i
 rescue
-  return 1
+  return 2
 end
 
-def setup_log4j config
-  path = config.delete(:path) || '/var/log/anschel4j.log'
-  pattern = config.delete(:pattern) || '[%d] %p %m (%c)%n'
-  Tempfile.open('anschel_log4j') do |f|
-    log4j = %Q|
-      log4j.rootLogger=INFO, A1
-      log4j.appender.A1=org.apache.log4j.RollingFileAppender
-      log4j.appender.A1.File=%s
-      log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-      log4j.appender.A1.layout.ConversionPattern=%s
-    | % [ path, pattern ]
-    f.write log4j.gsub(/^\s+/,'')
-    f.rewind
-    org.apache.log4j.PropertyConfigurator.configure(f.path)
-  end
-end
-
-Thread.abort_on_exception = true
-
 require_relative 'anschel/main'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anschel
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Sean Clemmer