nfagent 0.9.19 → 0.9.20

data/History.txt

@@ -1,4 +1,4 @@
-== 0.0.1 2009-09-22
+=== 0.0.1 2011-02-03
 
 * 1 major enhancement:
   * Initial release

data/Manifest.txt

@@ -1,29 +1,45 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
 Rakefile
+script/console
+script/destroy
+script/generate
+test/test_chunk.rb
+test/test_chunk_handler.rb
+test/test_client.rb
+test/test_config.rb
+test/test_helper.rb
+test/test_mapper_proxy.rb
+test/test_nfagent.rb
+test/test_payload.rb
+test/test_plugin.rb
+test/config
+test/plugins/my_mapper.rb
 lib/nfagent.rb
 lib/nfagent
-lib/nfagent/cli.rb
 lib/nfagent/base64.rb
-lib/nfagent/encoder.rb
-lib/nfagent/server.rb
-lib/nfagent/chunk_handler.rb
-lib/nfagent/log.rb
-lib/nfagent/info.rb
-lib/nfagent/event.rb
-lib/nfagent/payload.rb
-lib/nfagent/poller.rb
 lib/nfagent/chunk.rb
+lib/nfagent/chunk_handler.rb
+lib/nfagent/cli.rb
 lib/nfagent/client.rb
 lib/nfagent/client_response.rb
 lib/nfagent/config.rb
+lib/nfagent/encoder.rb
+lib/nfagent/event.rb
+lib/nfagent/info.rb
+lib/nfagent/log.rb
+lib/nfagent/mapper_proxy.rb
+lib/nfagent/payload.rb
+lib/nfagent/plugin.rb
+lib/nfagent/poller.rb
+lib/nfagent/server.rb
+lib/nfagent/submitter.rb
 lib/nfagent/tail.rb
 lib/nfagent/tests.rb
-lib/nfagent/submitter.rb
 bin/nfagent
 bin/squid_log_writer
-PostInstall.txt
-History.txt
-Manifest.txt
-README.txt
 nfagent.conf
 nfagent.init.ubuntu.txt
 nfagent.init.redhat.txt

data/PostInstall.txt

@@ -1 +1,7 @@
+
 For more information on nfagent, see http://nfagent.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/{README.txt → README.rdoc}

@@ -1,40 +1,32 @@
-= NetFox Agent
+= nfagent
 
-* http://netfox.com
+* http://github.com/#{github_username}/#{project_name}
 
 == DESCRIPTION:
 
-Logging Agent for NetFox Online
+FIX (describe your package)
 
 == FEATURES/PROBLEMS:
 
+* FIX (list of features or problems)
 
 == SYNOPSIS:
 
+  FIX (code sample of usage)
 
 == REQUIREMENTS:
 
-* Event Machine
-* SV Util
+* FIX (list of requirements)
 
 == INSTALL:
 
-sudo gem install nfagent
-
-== DEVELOPERS:
-
-After checking out the source, run:
-
-  $ rake newb
-
-This task will install any missing dependencies, run the tests/specs,
-and generate the RDoc.
+* FIX (sudo gem install, anything else)
 
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2010 FIX
+Copyright (c) 2011 FIXME full name
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -53,4 +45,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,31 +1,28 @@
-%w[rubygems rake rake/clean hoe fileutils newgem rubigen].each { |f| require f }
-require File.dirname(__FILE__) + '/lib/nfagent'
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/nfagent'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
 
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
-$hoe = Hoe.spec('nfagent') do |p|
-  p.version = NFAgent::VERSION
-  p.summary = "Logging Agent for NetFox Online"
-  p.developer('Daniel Draper', 'daniel@netfox.com')
-  p.changes              = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  p.post_install_message = 'PostInstall.txt'
-  p.rubyforge_name       = p.name
-  p.extra_deps         = [
-    ['svutil','>= 0.0.12'], ['eventmachine', '>= 0.12.8']
-  ]
-  p.extra_dev_deps = [
-    ['newgem', ">= #{::Newgem::VERSION}"]
+$hoe = Hoe.spec 'nfagent' do
+  self.developer('Daniel Draper', 'daniel@netfox.com')
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps         = [
+    ['svutil','>= 0.0.13'], ['eventmachine', '>= 0.12.8'], ['squiggle']
   ]
-  p.clean_globs |= %w[**/.DS_Store tmp *.log]
-  path = (p.rubyforge_name == p.name) ? p.rubyforge_name : "\#{p.rubyforge_name}/\#{p.name}"
-  p.remote_rdoc_dir = File.join(path.gsub(/^#{p.rubyforge_name}\/?/,''), 'rdoc')
-  p.rsync_args = '-av --delete --ignore-errors'
-  p.readme_file = "README.txt"
-  p.spec_extras[:default_executable] = 'nfagent'
+
 end
 
-require 'newgem/tasks' # load /tasks/*.rake
+require 'newgem/tasks'
 Dir['tasks/**/*.rake'].each { |t| load t }
 
 # TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
 # task :default => [:spec, :features]

data/lib/nfagent.rb

@@ -3,6 +3,7 @@ $:.unshift(File.dirname(__FILE__)) unless
 
 require 'rubygems'
 require 'svutil'
+require 'squiggle'
 
 require 'fileutils'
 require 'logger'
@@ -17,6 +18,8 @@ require 'nfagent/chunk'
 require 'nfagent/client'
 require 'nfagent/client_response'
 require 'nfagent/chunk_handler'
+require 'nfagent/mapper_proxy'
+require 'nfagent/plugin'
 require 'nfagent/submitter'
 require 'nfagent/encoder'
 require 'nfagent/config'
@@ -31,5 +34,5 @@ require 'nfagent/cli'
 require 'nfagent/tests'
 
 module NFAgent
-  VERSION = '0.9.19'
+  VERSION = '0.9.20'
 end

data/lib/nfagent/chunk.rb

@@ -2,44 +2,54 @@ require 'zlib'
 require 'digest'
 
 module NFAgent
-  class Chunk
+  class ChunkExpired < StandardError; end
+  class ChunkFull < StandardError; end
+  class DayBoundary < StandardError; end
+
+  class Chunk < Array
     attr_reader :created_at
+    attr_reader :max_size
 
-    ::DEFAULT_TIME_OUT = 60
+    DEFAULT_MAX_SIZE = 500
 
-    def initialize(max_size = 500)
+    def initialize(max_size = DEFAULT_MAX_SIZE)
       @max_size = max_size
       @created_at = Time.now
-      @array = []
-      @submitter = Submitter.new(Config.client_key)
     end
 
     def <<(line)
-      @array << line
+      raise ChunkExpired if expired?
+      raise ChunkFull if full?
+      raise DayBoundary if Time.now.day != self.created_at.day
+      super(line)
     end
 
     def full?
-      @array.size >= @max_size
+      self.size >= @max_size
     end
 
     def expired?
-      (Time.now - @created_at > ::DEFAULT_TIME_OUT) && !@array.empty?
+      (Time.now - @created_at > Config.chunk_time_out) && !self.empty?
     end
 
-    # TODO: Is this the right place for compression, encoding and check summing? Perhaps it should go into the submitter to that it can be deferred
-    def dump
+    def dump(key = nil)
       Payload.new do |payload|
-        Log.info("Dumping payload from chunk (#{@array.size} lines)")
-        payload.line_count = @array.size
+        Log.info("Dumping payload from chunk (#{self.size} lines)")
+        payload.line_count = self.size
         payload.chunk_expired = expired?
-        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
+        payload.key = key
+        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(self.join("\n"), Zlib::BEST_COMPRESSION))
         payload.checksum = Digest::SHA1.hexdigest(payload.data)
       end
     end
 
-    def clear
-      @array.clear
-      @created_at = Time.now
+    def submit(key = nil)
+      submitter = Submitter.new(self.dump(key))
+      submitter.errback { |payload|
+        payload.write_to_disk(Config.dump_dir)
+      }
+      submitter.perform
+      # Callback and remove from chunk group
     end
   end
 end

data/lib/nfagent/chunk_handler.rb

@@ -1,40 +1,65 @@
 module NFAgent
   class ChunkHandler
 
-    # TODO: Rename this to Controller later
-    def initialize(chunk_size = 500)
-      @chunk = Chunk.new(chunk_size)
+    attr_accessor :chunk_group
+
+    def initialize(options = {})
+      @chunk_size = options[:chunk_size] || 500
+      @parser = options[:parser] || Squiggle::SquidStandardParser.new(Config.time_zone)
+      @chunk_group = {}
+      class << @chunk_group
+        def fetch!(key, new_chunk)
+          if self.has_key?(key)
+            self.fetch(key)
+          else
+            self[key] = new_chunk
+            new_chunk
+          end
+        end
+      end
     end
 
     def append(line)
-      # if current day is > day of last entry on current_chunk 
-      # then submit and reset the chunk before adding the line
-      current_day = Time.now.day
-      if current_day != @chunk.created_at.day
-        Log.info("Expiring chunk due to date rollover")
-        reset_chunk
+      if Config.parse == 'locally'
+        parsed = @parser.parse(line)
+        return if parsed.invalid?
+        if Config.mode == 'multi'
+          key = MapperProxy.find_account_id(parsed.username, parsed.client_ip)
+          # TODO: Still appending line as string until Server API has been updated
+          return append2(line, key)
+        end
+      end
+      # TODO: rename append2
+      append2(line)
+    end
+
+    def append2(line, key = nil)
+      key ||= 'all'
+      begin
+        chunk = @chunk_group.fetch!(key, Chunk.new(@chunk_size))
+        chunk << line
+      rescue ChunkExpired, ChunkFull
+        reset_chunk(key)
       end
-      @chunk << line
     end
 
     def check_full_or_expired
-      if @chunk.full?
-        Log.info("Chunk Full: Resetting...")
-        reset_chunk
-      elsif @chunk.expired?
-        Log.info("Chunk Expired: Resetting...")
-        reset_chunk
+      @chunk_group.each_pair do |key, chunk|
+        if chunk.full?
+          Log.info("Chunk Full: Resetting...")
+          reset_chunk(key)
+        elsif chunk.expired?
+          Log.info("Chunk Expired: Resetting...")
+          reset_chunk(key)
+        end
       end
     end
 
     private
-      def reset_chunk
-        submitter = Submitter.new(@chunk.dump)
-        submitter.errback { |payload|
-          payload.write_to_disk(Config.dump_dir)
-        }
-        @chunk.clear
-        submitter.perform
+      def reset_chunk(key = nil)
+        key ||= 'all'
+        chunk = @chunk_group.delete(key)
+        chunk.submit
       end
   end
 end

data/lib/nfagent/client.rb

@@ -1,6 +1,6 @@
 module NFAgent
   class Client
-     SERVICE_HOST = "collector.service.netfox.com"
+    SERVICE_HOST = "collector.service.netfox.com"
 
     def self.post(end_point, data_hash)
       proxy_class = Net::HTTP::Proxy(Config.http_proxy_host, Config.http_proxy_port, Config.http_proxy_user, Config.http_proxy_password)
@@ -8,7 +8,7 @@ module NFAgent
       proxy_class.start(SERVICE_HOST, 80) do |http|
         http.read_timeout = 120 # 2 minutes TODO: Make this a config option with 120 as default
         req = Net::HTTP::Post.new("/#{end_point}")
-        req.set_form_data(data_hash.merge("key" => Config.client_key))
+        req.set_form_data({"key" => Config.client_key}.merge(data_hash))
         ClientResponse.new do |resp|
           resp.response, resp.message = http.request(req)
         end

data/lib/nfagent/config.rb

@@ -8,11 +8,41 @@ module NFAgent
       @@test_mode
     end
 
+    # Config Options
+    # client_key: String, the access key for the client (for the account in normal mode or for a partner in multi mode)
+    # dump_dir: String, the directory path of a local spool location
+    # pid_file: String, path of process ID file
+    # mode: (optional, default: 'normal') String, either 'normal' or 'multi' - can be left blank
+    # mapping: Class, this is a plugin class which must be stored in a file in the directory /etc/nfagent/plugins/
+    # parse: (optional, default: 'remotely'): String, either 'remotely' or 'locally'
+    #
     class << self
+      def set_defaults
+        default('mode', 'normal')
+        default('parse', 'remotely')
+        default('chunk_time_out', 60)
+        default('time_zone', 'UTC')
+        default('plugin_directory', '/etc/nfagent/plugins/')
+      end
+
       def validate
         unless dump_dir and File.exists?(dump_dir) and File.directory?(dump_dir)
           raise "Dump dir (#{dump_dir}) must exist and be a directory"
         end
+        # Mode
+        unless %w(normal multi).include?(mode)
+          raise "Invalid mode: must be one of 'normal' or 'multi'"
+        end
+        if mode == 'multi' && mapper.blank?
+          raise "Multi mode requires a mapper to be set"
+        end
+        if mode == 'multi' && parse != 'locally'
+          raise "Multi mode requires that parsing be done locally (set parse = 'locally')"
+        end
+        # Parse
+        unless %w(remotely locally).include?(parse)
+          raise "Invalid parse option: Must be one of 'remotely' or 'locally'"
+        end
         super
       end
 
@@ -30,6 +60,9 @@ module NFAgent
           opts.on("-T", "--test", "Run connection tests") do
             @@test_mode = true
           end
+          opts.on("-P", "--parse", "Parse locally before submitting") do
+            Config.parse_locally = true
+          end
         end
       end
     end

data/lib/nfagent/mapper_proxy.rb

@@ -0,0 +1,18 @@
+module NFAgent
+  class MapperProxy
+    class << self
+      def instance
+        return @instance if @instance
+        raise "No Mapper Set" if Config.mapper.blank?
+        @instance = Config.mapper.constantize.new
+      end
+
+      # TODO: Can we delegate?
+      def find_account_id(username, client_ip)
+        instance.find_account_id(username, client_ip)
+      end
+
+      # TODO: before shutdown
+    end
+  end
+end

data/lib/nfagent/payload.rb

@@ -1,6 +1,6 @@
 
 module NFAgent
-  class Payload < Struct.new(:data, :checksum, :filename, :line_count, :chunk_expired)
+  class Payload < Struct.new(:data, :checksum, :filename, :line_count, :chunk_expired, :key)
     def initialize
       yield self
     end
@@ -23,12 +23,14 @@ module NFAgent
     end
 
     def to_hash
-      {
+      ret = {
         "payload" => self.data,
         "checksum" => self.checksum,
         "line_count" => self.line_count,
         "chunk_expired" => self.chunk_expired
       }
+      ret["key"] = self.key unless self.key.blank?
+      ret
     end
 
     def write_to_disk(directory)

data/lib/nfagent/plugin.rb

@@ -0,0 +1,13 @@
+module NFAgent
+  class Plugin
+    class << self
+      def load_plugins
+        unless Config.plugin_directory.blank?
+          Dir.glob(File.join(Config.plugin_directory, "*.rb")).each do |file|
+            load(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nfagent/server.rb

@@ -2,6 +2,8 @@ module NFAgent
   class Server
     def run
       Log.info("Starting up")
+      NFAgent::Plugin.load_plugins
+
       chunk_handler = ChunkHandler.new
       poller = Poller.new
 

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/nfagent.rb'}"
+puts "Loading nfagent gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/config

@@ -0,0 +1,5 @@
+client_key = 1234
+dump_dir = test/sandbox/dumps
+log_file = test/sandbox/debug
+pid_file = test/sandbox/nfagent.pid
+plugin_directory = test/plugins/

data/test/plugins/my_mapper.rb

@@ -0,0 +1,17 @@
+class MyMapper
+  def initialize
+    @hash = {
+      'dan' => 'acme',
+      'paul' => 'jetson',
+      'mike' => 'acme'
+    }
+  end
+
+  def before_shutdown
+    # Put any cleanup code in here
+  end
+
+  def find_account_id(username, client_ip)
+    @hash[username]
+  end
+end

data/test/test_chunk.rb

@@ -0,0 +1,80 @@
+
+require 'test/test_helper'
+
+class TestChunk < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.load_and_parse
+    @logline = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico - NONE/- text/html"
+  end
+
+  test "initialize" do
+    chunk = NFAgent::Chunk.new
+    assert_equal 500, chunk.max_size
+    chunk = NFAgent::Chunk.new(600)
+    assert_equal 600, chunk.max_size
+  end
+
+  test "full" do
+    chunk = NFAgent::Chunk.new(20)
+    19.times do
+      chunk << @logline
+    end
+    assert !chunk.full?
+    chunk << @logline
+    assert chunk.full?
+    assert_raises(NFAgent::ChunkFull) { chunk << @logline }
+  end
+
+  test "won't expire if empty" do
+    NFAgent::Config.chunk_time_out = 1
+    NFAgent::Config.validate
+    chunk = NFAgent::Chunk.new
+    assert !chunk.expired?
+    Timecop.freeze(Time.now + 61) do
+      assert !chunk.expired?
+    end
+  end
+
+  test "expired" do
+    chunk = NFAgent::Chunk.new
+    chunk << @logline
+    assert !chunk.expired?
+    Timecop.freeze(Time.now + 61) do
+      assert chunk.expired?
+      assert_raises(NFAgent::ChunkExpired) { chunk << @logline }
+    end
+  end
+
+  test "day boundary" do
+    d = Date.today
+    Timecop.travel(Time.local(2011, 1, 10, 23, 59, 58)) do
+      chunk = NFAgent::Chunk.new
+      Timecop.travel(Time.local(2011, 1, 11, 0, 0, 10)) do
+        assert_raises(NFAgent::DayBoundary) { chunk << @logline }
+      end
+    end
+  end
+
+  test "submit" do
+    NFAgent::Submitter.any_instance.expects(:perform)
+    chunk = NFAgent::Chunk.new
+    chunk.submit
+  end
+
+  test "dump" do
+    Zlib::Deflate.expects(:deflate).returns("x\234\313\316*/\314HO-\312N\317(J\005\000&~\005u").times(2)
+    chunk = NFAgent::Chunk.new
+    chunk << @logline
+    payload = chunk.dump
+    assert_equal 1, payload.line_count
+    assert_instance_of String, payload.data
+    # Dump with key
+    payload = chunk.dump('acme')
+    assert_equal 'acme', payload.key
+  end
+
+  test "dump from LogLine" do
+    flunk
+  end
+end

data/test/test_chunk_handler.rb

@@ -0,0 +1,97 @@
+
+require 'test/test_helper'
+
+class TestChunkHandler < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.load_and_parse
+    NFAgent::Config.plugin_directory = File.dirname(__FILE__) + '/../test/plugins/' 
+    NFAgent::Plugin.load_plugins
+    @logline = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico dan NONE/- text/html"
+    @logline2 = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico paul NONE/- text/html"
+  end
+
+  test "append line" do
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert_equal 1, chunk_handler.chunk_group['all'].size
+    assert_instance_of String, chunk_handler.chunk_group['all'][-1]
+  end
+
+  test "append with local parsing" do
+    NFAgent::Config.parse = 'locally'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert_equal 1, chunk_handler.chunk_group['all'].size
+    assert_instance_of Squiggle::LogLine, chunk_handler.chunk_group['all'][-1]
+  end
+
+  test "append with multi" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Config.validate
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    assert chunk_handler.chunk_group.has_key?('acme')
+    assert_equal 1, chunk_handler.chunk_group['acme'].size
+    chunk_handler.append(@logline2)
+    assert chunk_handler.chunk_group.has_key?('acme')
+    assert chunk_handler.chunk_group.has_key?('jetson')
+    assert_equal 1, chunk_handler.chunk_group['acme'].size
+    assert_equal 1, chunk_handler.chunk_group['jetson'].size
+  end
+
+  test "reset chunk after expiry" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Config.validate
+    NFAgent::Submitter.any_instance.expects(:perform).times(2)
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(30) do
+      chunk_handler.append(@logline2)
+      assert_equal 1, chunk_handler.chunk_group['acme'].size
+      assert_equal 1, chunk_handler.chunk_group['jetson'].size
+
+      Timecop.travel(31) do
+        chunk_handler.check_full_or_expired
+        assert !chunk_handler.chunk_group.has_key?('acme')
+        assert_equal 1, chunk_handler.chunk_group['jetson'].size
+
+        Timecop.travel(31) do
+          chunk_handler.check_full_or_expired
+          assert !chunk_handler.chunk_group.has_key?('acme')
+          assert !chunk_handler.chunk_group.has_key?('jestson')
+        end
+      end
+    end
+  end
+
+  test "reset chunk after full wuth check_full_or_expired" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Config.validate
+    NFAgent::Submitter.any_instance.expects(:perform).times(2)
+    chunk_handler = NFAgent::ChunkHandler.new(:chunk_size => 10)
+    9.times { chunk_handler.append(@logline) }
+    9.times { chunk_handler.append(@logline2) }
+    assert_equal 9, chunk_handler.chunk_group['acme'].size
+    assert_equal 9, chunk_handler.chunk_group['jetson'].size
+    chunk_handler.append(@logline)
+    chunk_handler.check_full_or_expired
+    assert !chunk_handler.chunk_group.has_key?('acme')
+    chunk_handler.append(@logline2)
+    chunk_handler.check_full_or_expired
+    assert !chunk_handler.chunk_group.has_key?('jetson')
+  end
+
+  test "fail an invalid logline in local parse mode" do
+    NFAgent::Config.parse = 'locally'
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append("") # Invalid logline
+    assert chunk_handler.chunk_group['all'].nil?
+  end
+end

data/test/test_client.rb

@@ -0,0 +1,20 @@
+
+require 'test/test_helper'
+
+class TestClient < ActiveSupport::TestCase
+  setup do
+  end
+
+  test "post data with configured client key" do
+    Net::HTTP::Post.any_instance.expects(:set_form_data).with({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => '1234' })
+    hash = { 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false }
+    NFAgent::Client.post(:collector, hash)
+  end
+
+  test "post data with key passed in data hash" do
+    Net::HTTP::Post.any_instance.expects(:set_form_data).with({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => 'abcd' })
+    hash = { 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => 'abcd' }
+    NFAgent::Client.post(:collector, hash)
+  end
+
+end

data/test/test_config.rb

@@ -0,0 +1,40 @@
+
+require 'test/test_helper'
+
+class TestConfig < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.config_file = "test/config"
+    NFAgent::Config.load_and_parse
+  end
+
+  test "defaults" do
+    assert_equal "normal", NFAgent::Config.mode
+    assert_equal "remotely", NFAgent::Config.parse
+    assert_equal 60, NFAgent::Config.chunk_time_out
+    assert_equal 'UTC',  NFAgent::Config.time_zone
+    assert_equal '/etc/nfagent/plugins/', NFAgent::Config.plugin_directory
+  end
+
+  test "validates valid mode" do
+    NFAgent::Config.mode = 'some stupid thing'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+  end
+
+  test "validates mapping with multi" do
+    NFAgent::Config.mode = 'multi'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.mapper = 'AccountMapper'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.parse = 'locally'
+    assert NFAgent::Config.validate
+  end
+
+  test "validates valid parse option" do
+    NFAgent::Config.parse = 'some stupid thing'
+    assert_raises(RuntimeError) { NFAgent::Config.validate }
+    NFAgent::Config.parse = 'locally'
+    assert NFAgent::Config.validate
+    NFAgent::Config.parse = 'remotely'
+    assert NFAgent::Config.validate
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+
+require 'mocha'
+require 'timecop'
+require 'active_support'
+require File.dirname(__FILE__) + '/../lib/nfagent'
+

data/test/test_mapper_proxy.rb

@@ -0,0 +1,20 @@
+
+require 'test/test_helper'
+
+class TestMapperProxy < ActiveSupport::TestCase
+  setup do
+    NFAgent::Config.plugin_directory = File.dirname(__FILE__) + '/../test/plugins/' 
+  end
+
+  test "instantiate just once" do
+    MyMapper.expects(:new).at_most_once
+    NFAgent::Config.mapper = nil
+    NFAgent::MapperProxy.instance
+    NFAgent::MapperProxy.instance
+    NFAgent::MapperProxy.instance
+  end
+
+  test "mapper method" do
+    assert_equal 'acme', NFAgent::MapperProxy.find_account_id('dan', '192.168.0.10')
+  end
+end

data/test/test_nfagent.rb

@@ -0,0 +1,8 @@
+
+require 'test/test_helper'
+
+class NFAgentTest < ActiveSupport::TestCase
+  test "just return true" do
+    assert true
+  end
+end

data/test/test_payload.rb

@@ -0,0 +1,26 @@
+
+require 'test/test_helper'
+
+class TestPayload < ActiveSupport::TestCase
+  setup do
+  end
+
+  test "to_hash" do
+    payload = NFAgent::Payload.new do |p|
+      p.data = "data"
+      p.checksum = "checksum"
+      p.line_count = 10
+      p.chunk_expired = false
+    end
+    assert_equal({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false }, payload.to_hash)
+    # Check key
+    payload = NFAgent::Payload.new do |p|
+      p.data = "data"
+      p.checksum = "checksum"
+      p.line_count = 10
+      p.chunk_expired = false
+      p.key = '1234'
+    end
+    assert_equal({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => '1234' }, payload.to_hash)
+  end
+end

data/test/test_plugin.rb

@@ -0,0 +1,13 @@
+
+require 'test/test_helper'
+
+class TestPlugin < ActiveSupport::TestCase
+  setup do
+  end
+
+  test "load" do
+    assert_raise(NameError) { "MissingMapper".constantize }
+    NFAgent::Plugin.load_plugins
+    "MyMapper".constantize
+  end
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 19
-  version: 0.9.19
+  - 20
+  version: 0.9.20
 platform: ruby
 authors: 
 - Daniel Draper
@@ -14,8 +14,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-14 00:00:00 +10:30
-default_executable: nfagent
+date: 2011-02-06 00:00:00 +10:30
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: svutil
@@ -27,8 +27,8 @@ dependencies:
         segments: 
         - 0
         - 0
-        - 12
-        version: 0.0.12
+        - 13
+        version: 0.0.13
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -46,23 +46,35 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: newgem
+  name: squiggle
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
         segments: 
-        - 1
-        - 5
-        - 3
-        version: 1.5.3
-  type: :development
+        - 0
+        version: "0"
+  type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: rubyforge
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -72,8 +84,8 @@ dependencies:
         - 0
         version: 2.6.0
   type: :development
-  version_requirements: *id004
-description: Logging Agent for NetFox Online
+  version_requirements: *id005
+description: FIX (describe your package)
 email: 
 - daniel@netfox.com
 executables: 
@@ -82,49 +94,64 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
-- PostInstall.txt
 - History.txt
 - Manifest.txt
-- README.txt
+- PostInstall.txt
 - nfagent.init.ubuntu.txt
 - nfagent.init.redhat.txt
 files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
 - Rakefile
+- script/console
+- script/destroy
+- script/generate
+- test/test_chunk.rb
+- test/test_chunk_handler.rb
+- test/test_client.rb
+- test/test_config.rb
+- test/test_helper.rb
+- test/test_mapper_proxy.rb
+- test/test_nfagent.rb
+- test/test_payload.rb
+- test/test_plugin.rb
+- test/config
+- test/plugins/my_mapper.rb
 - lib/nfagent.rb
-- lib/nfagent/cli.rb
 - lib/nfagent/base64.rb
-- lib/nfagent/encoder.rb
-- lib/nfagent/server.rb
-- lib/nfagent/chunk_handler.rb
-- lib/nfagent/log.rb
-- lib/nfagent/info.rb
-- lib/nfagent/event.rb
-- lib/nfagent/payload.rb
-- lib/nfagent/poller.rb
 - lib/nfagent/chunk.rb
+- lib/nfagent/chunk_handler.rb
+- lib/nfagent/cli.rb
 - lib/nfagent/client.rb
 - lib/nfagent/client_response.rb
 - lib/nfagent/config.rb
+- lib/nfagent/encoder.rb
+- lib/nfagent/event.rb
+- lib/nfagent/info.rb
+- lib/nfagent/log.rb
+- lib/nfagent/mapper_proxy.rb
+- lib/nfagent/payload.rb
+- lib/nfagent/plugin.rb
+- lib/nfagent/poller.rb
+- lib/nfagent/server.rb
+- lib/nfagent/submitter.rb
 - lib/nfagent/tail.rb
 - lib/nfagent/tests.rb
-- lib/nfagent/submitter.rb
 - bin/nfagent
 - bin/squid_log_writer
-- PostInstall.txt
-- History.txt
-- Manifest.txt
-- README.txt
 - nfagent.conf
 - nfagent.init.ubuntu.txt
 - nfagent.init.redhat.txt
 has_rdoc: true
-homepage: http://netfox.com
+homepage: http://github.com/#{github_username}/#{project_name}
 licenses: []
 
 post_install_message: PostInstall.txt
 rdoc_options: 
 - --main
-- README.txt
+- README.rdoc
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -147,6 +174,14 @@ rubyforge_project: nfagent
 rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
-summary: Logging Agent for NetFox Online
-test_files: []
-
+summary: FIX (describe your package)
+test_files: 
+- test/test_chunk.rb
+- test/test_chunk_handler.rb
+- test/test_client.rb
+- test/test_config.rb
+- test/test_helper.rb
+- test/test_mapper_proxy.rb
+- test/test_nfagent.rb
+- test/test_payload.rb
+- test/test_plugin.rb