nfagent 0.9.31 → 0.9.50

data/lib/nfagent/event.rb

@@ -13,7 +13,7 @@ module NFAgent
       if data && data.length > 10
         @handler.append(data)
       end
-      send_data('OK')
+      #send_data('OK')
     end
 
     def unbind

data/lib/nfagent/mapper_proxy.rb

@@ -0,0 +1,18 @@
+module NFAgent
+  class MapperProxy
+    class << self
+      def instance
+        return @instance if @instance
+        raise "No Mapper Set" if Config.mapper.blank?
+        @instance = Object.const_get(Config.mapper).new
+      end
+
+      # TODO: Can we delegate?
+      def find_account_id(username, client_ip)
+        instance.find_account_id(username, client_ip)
+      end
+
+      # TODO: before shutdown
+    end
+  end
+end

data/lib/nfagent/object_extra.rb

@@ -0,0 +1,5 @@
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end

data/lib/nfagent/payload.rb

@@ -1,6 +1,6 @@
 
 module NFAgent
-  class Payload < Struct.new(:data, :checksum, :filename, :line_count, :chunk_expired)
+  class Payload < Struct.new(:data, :checksum, :filename, :line_count, :chunk_expired, :key)
     def initialize
       yield self
     end
@@ -23,16 +23,19 @@ module NFAgent
     end
 
     def to_hash
-      {
+      ret = {
         "payload" => self.data,
         "checksum" => self.checksum,
         "line_count" => self.line_count,
         "chunk_expired" => self.chunk_expired
       }
+      ret["key"] = self.key unless self.key.blank?
+      ret
     end
 
     def write_to_disk(directory)
-      File.open(File.join(directory, "#{self.checksum}-#{self.attempt}"), "w") do |file|
+      filename = [ self.checksum, self.attempt, self.key ].compact.join("-")
+      File.open(File.join(directory, filename), "w") do |file|
         file << self.data
       end
     end
@@ -48,14 +51,14 @@ module NFAgent
       filename && File.exists?(lockfile)
     end
 
-    def self.read_from_file(filename)
+    def self.read_from_file(filename, dir = Config.dump_dir)
       # Ensure the file is only relative
       filename = File.basename(filename)
       self.new do |payload|
         payload.filename = filename
-        payload.checksum, payload.attempt = filename.split("-")
+        payload.checksum, payload.attempt, payload.key = filename.split("-")
         payload.data = ""
-        ref = File.join(Config.dump_dir, filename)
+        ref = File.join(dir, filename)
         File.open(ref, "r") do |file|
           payload.data << file.read
         end
@@ -68,7 +71,8 @@ module NFAgent
 
     def try_again_later
       # TODO: Move the file to a new name with a later timetamp
-      FileUtils.mv(File.join(Config.dump_dir, self.filename), File.join(Config.dump_dir, "#{self.checksum}-#{self.attempt}"))
+      new_filename = [ self.checksum, self.attempt, self.key ].compact.join("-")
+      FileUtils.mv(File.join(Config.dump_dir, self.filename), File.join(Config.dump_dir, new_filename))
     end
 
     private

data/lib/nfagent/plugin.rb

@@ -0,0 +1,14 @@
+module NFAgent
+  class Plugin
+    class << self
+      def load_plugins
+        if Config.plugin_directory && !Config.plugin_directory.empty? && File.exists?(Config.plugin_directory)
+          Dir.glob(File.join(Config.plugin_directory, "*.rb")).each do |file|
+            Log.info("Loading plugin: #{file}")
+            load(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nfagent/server.rb

@@ -2,6 +2,11 @@ module NFAgent
   class Server
     def run
       Log.info("Starting up")
+      NFAgent::Plugin.load_plugins
+
+      Log.info("Parsing #{Config.parse}")
+      Log.info("Mode set to #{Config.mode}")
+
       chunk_handler = ChunkHandler.new
       poller = Poller.new
 

data/lib/nfagent/submitter.rb

@@ -15,6 +15,7 @@ module NFAgent
         response = Client.post(:collector, @payload.to_hash)
         if response.ok?
           succeed(@payload)
+          Log.info("Submitted #{@payload.line_count} lines")
         else
           Log.error "Submission Failed: #{response.message}"
           fail(@payload)

data/nfagent.conf

@@ -1,4 +1,4 @@
-client_key = 1234
-dump_dir = /tmp/dumps
-log_file = /tmp/debug
-pid_file = /tmp/nfagent.pid
+client_key = hOFxKEgYEBf1Ni8WqJ02
+dump_dir = sandbox/dumps
+log_file = sandbox/debug
+pid_file = sandbox/nfagent.pid

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/nfagent.rb'}"
+puts "Loading nfagent gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/config

@@ -0,0 +1,5 @@
+client_key = 1234
+dump_dir = sandbox/dumps
+log_file = sandbox/debug
+pid_file = sandbox/nfagent.pid
+plugin_directory = test/plugins/

data/test/plugins/my_mapper.rb

@@ -0,0 +1,18 @@
+class MyMapper
+  def initialize
+    @hash = {
+      'dan' => 'acme',
+      'paul' => 'jetson',
+      'mike' => 'acme'
+    }
+  end
+
+  def before_shutdown
+    # Put any cleanup code in here
+  end
+
+  def find_account_id(username, client_ip)
+    raise NFAgent::IgnoreLine if username == 'andrew'
+    @hash[username]
+  end
+end

data/test/test_chunk.rb

@@ -0,0 +1,79 @@
+
+require 'test/test_helper'
+
+class TestChunk < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.init
+    @logline = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico - NONE/- text/html"
+  end
+
+  test "initialize" do
+    chunk = NFAgent::Chunk.new
+    assert_equal 500, chunk.max_size
+    chunk = NFAgent::Chunk.new(600)
+    assert_equal 600, chunk.max_size
+  end
+
+  test "full" do
+    chunk = NFAgent::Chunk.new(20)
+    19.times do
+      chunk << @logline
+    end
+    assert !chunk.full?
+    chunk << @logline
+    assert chunk.full?
+    assert_raises(NFAgent::ChunkFull) { chunk << @logline }
+  end
+
+  test "won't expire if empty" do
+    NFAgent::Config.chunk_time_out = 1
+    chunk = NFAgent::Chunk.new
+    assert !chunk.expired?
+    Timecop.freeze(Time.now + 61) do
+      assert !chunk.expired?
+    end
+  end
+
+  test "expired" do
+    chunk = NFAgent::Chunk.new
+    chunk << @logline
+    assert !chunk.expired?
+    Timecop.freeze(Time.now + 61) do
+      assert chunk.expired?
+      assert_raises(NFAgent::ChunkExpired) { chunk << @logline }
+    end
+  end
+
+  test "day boundary" do
+    d = Date.today
+    Timecop.travel(Time.local(2011, 1, 10, 23, 59, 58)) do
+      chunk = NFAgent::Chunk.new
+      Timecop.travel(Time.local(2011, 1, 11, 0, 0, 10)) do
+        assert_raises(NFAgent::DayBoundary) { chunk << @logline }
+      end
+    end
+  end
+
+  test "submit" do
+    EM.expects(:defer)
+    chunk = NFAgent::Chunk.new
+    chunk.submit
+  end
+
+  test "dump" do
+    Zlib::Deflate.expects(:deflate).returns("x\234\313\316*/\314HO-\312N\317(J\005\000&~\005u").times(2)
+    chunk = NFAgent::Chunk.new
+    chunk << @logline
+    payload = chunk.dump
+    assert_equal 1, payload.line_count
+    assert_instance_of String, payload.data
+    # Dump with key
+    payload = chunk.dump('acme')
+    assert_equal 'acme', payload.key
+  end
+
+  test "dump from LogLine" do
+    pending
+  end
+end

data/test/test_chunk_handler.rb

@@ -0,0 +1,131 @@
+
+require 'test/test_helper'
+
+class TestChunkHandler < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.init
+    NFAgent::Config.plugin_directory = File.dirname(__FILE__) + '/../test/plugins/' 
+    NFAgent::Plugin.load_plugins
+    @logline = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico dan NONE/- text/html"
+    @logline2 = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico paul NONE/- text/html"
+    @ignored_logline = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico andrew NONE/- text/html"
+  end
+
+  test "append line" do
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert_equal 1, chunk_handler.chunk_group['all'].size
+    assert_instance_of String, chunk_handler.chunk_group['all'][-1]
+  end
+
+  test "append with local parsing" do
+    NFAgent::Config.parse = 'locally'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert_equal 1, chunk_handler.chunk_group['all'].size
+    assert_instance_of Squiggle::LogLine, chunk_handler.chunk_group['all'][-1]
+  end
+
+  test "append with multi" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert chunk_handler.chunk_group.has_key?('acme')
+    assert_equal 1, chunk_handler.chunk_group['acme'].size
+    chunk_handler.append(@logline2)
+    assert chunk_handler.chunk_group.has_key?('acme')
+    assert chunk_handler.chunk_group.has_key?('jetson')
+    assert_equal 1, chunk_handler.chunk_group['acme'].size
+    assert_equal 1, chunk_handler.chunk_group['jetson'].size
+  end
+
+  test "mapper raises ignore line in multi mode" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@ignored_logline)
+    assert !chunk_handler.chunk_group.has_key?('acme')
+    assert !chunk_handler.chunk_group.has_key?('jetson')
+    assert chunk_handler.chunk_group['acme'].blank?
+    assert chunk_handler.chunk_group['jetson'].blank?
+  end
+
+  test "reset chunk after expiry in multi mode" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Chunk.any_instance.expects(:submit).at_least_once
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(30) do
+      chunk_handler.append(@logline2)
+      assert_equal 1, chunk_handler.chunk_group['acme'].size
+      assert_equal 1, chunk_handler.chunk_group['jetson'].size
+
+      Timecop.travel(31) do
+        chunk_handler.check_full_or_expired
+        assert !chunk_handler.chunk_group.has_key?('acme')
+        assert_equal 1, chunk_handler.chunk_group['jetson'].size
+
+        Timecop.travel(31) do
+          chunk_handler.check_full_or_expired
+          assert !chunk_handler.chunk_group.has_key?('acme')
+          assert !chunk_handler.chunk_group.has_key?('jestson')
+        end
+      end
+    end
+  end
+
+  test "reset chunk after full with check_full_or_expired in multi mode" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    EM.expects(:defer).times(2)
+    chunk_handler = NFAgent::ChunkHandler.new(:chunk_size => 10)
+    9.times { chunk_handler.append(@logline) }
+    9.times { chunk_handler.append(@logline2) }
+    assert_equal 9, chunk_handler.chunk_group['acme'].size
+    assert_equal 9, chunk_handler.chunk_group['jetson'].size
+    chunk_handler.append(@logline)
+    chunk_handler.check_full_or_expired
+    assert !chunk_handler.chunk_group.has_key?('acme')
+    chunk_handler.append(@logline2)
+    chunk_handler.check_full_or_expired
+    assert !chunk_handler.chunk_group.has_key?('jetson')
+  end
+
+  test "fail an invalid logline in local parse mode" do
+    NFAgent::Config.parse = 'locally'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append("") # Invalid logline
+    assert chunk_handler.chunk_group['all'].nil?
+  end
+
+  test "reset chunk is passed appropriate key in multi mode" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Chunk.any_instance.expects(:submit).with('acme')
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(61) do
+      chunk_handler.check_full_or_expired
+    end
+  end
+
+  test "reset chunk is passed nil key in normal mode" do
+    NFAgent::Config.parse = 'remotely'
+    NFAgent::Config.mode = 'normal'
+    NFAgent::Chunk.any_instance.expects(:submit).with(nil)
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(61) do
+      chunk_handler.check_full_or_expired
+    end
+  end
+
+end

data/test/test_client.rb

@@ -0,0 +1,20 @@
+
+require 'test/test_helper'
+
+class TestClient < ActiveSupport::TestCase
+  setup do
+  end
+
+  test "post data with configured client key" do
+    Net::HTTP::Post.any_instance.expects(:set_form_data).with({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => '1234' })
+    hash = { 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false }
+    NFAgent::Client.post(:collector, hash)
+  end
+
+  test "post data with key passed in data hash" do
+    Net::HTTP::Post.any_instance.expects(:set_form_data).with({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => 'abcd' })
+    hash = { 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => 'abcd' }
+    NFAgent::Client.post(:collector, hash)
+  end
+
+end

data/test/test_config.rb

@@ -0,0 +1,49 @@
+
+require 'test/test_helper'
+
+class TestConfig < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.attrs.clear
+  end
+
+  test "defaults" do
+    puts "HHHHHH"
+    p NFAgent::Config.attrs
+    assert_equal "normal", NFAgent::Config.mode
+    assert_equal "remotely", NFAgent::Config.parse
+    assert_equal 60, NFAgent::Config.chunk_timeout
+    assert_equal 'UTC',  NFAgent::Config.time_zone
+    assert_equal '/etc/nfagent/plugins/', NFAgent::Config.plugin_directory
+  end
+
+  test "validates valid mode" do
+    NFAgent::Config.mode = 'some stupid thing'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+  end
+
+  test "validates mapping with multi" do
+    NFAgent::Config.attrs.clear
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.init
+
+    NFAgent::Config.mode = 'multi'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.mapper = 'AccountMapper'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.parse = 'locally'
+    assert NFAgent::Config.validate
+  end
+
+  test "validates valid parse option" do
+    NFAgent::Config.attrs.clear
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.init
+
+    NFAgent::Config.parse = 'some stupid thing'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.parse = 'locally'
+    assert NFAgent::Config.validate
+    NFAgent::Config.parse = 'remotely'
+    assert NFAgent::Config.validate
+  end
+end