nfagent 0.9.31 → 0.9.50

data/History.txt

@@ -1,4 +1,4 @@
-== 0.0.1 2009-09-22
+=== 0.0.1 2011-02-03
 
 * 1 major enhancement:
   * Initial release

data/Manifest.txt

@@ -1,29 +1,46 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
 Rakefile
+script/console
+script/destroy
+script/generate
+test/test_chunk.rb
+test/test_chunk_handler.rb
+test/test_client.rb
+test/test_config.rb
+test/test_helper.rb
+test/test_mapper_proxy.rb
+test/test_nfagent.rb
+test/test_payload.rb
+test/test_plugin.rb
+test/config
+test/plugins/my_mapper.rb
 lib/nfagent.rb
 lib/nfagent
-lib/nfagent/cli.rb
 lib/nfagent/base64.rb
-lib/nfagent/encoder.rb
-lib/nfagent/server.rb
-lib/nfagent/chunk_handler.rb
-lib/nfagent/log.rb
-lib/nfagent/info.rb
-lib/nfagent/event.rb
-lib/nfagent/payload.rb
-lib/nfagent/poller.rb
 lib/nfagent/chunk.rb
+lib/nfagent/chunk_handler.rb
+lib/nfagent/cli.rb
 lib/nfagent/client.rb
 lib/nfagent/client_response.rb
 lib/nfagent/config.rb
+lib/nfagent/encoder.rb
+lib/nfagent/event.rb
+lib/nfagent/info.rb
+lib/nfagent/log.rb
+lib/nfagent/mapper_proxy.rb
+lib/nfagent/object_extra.rb
+lib/nfagent/payload.rb
+lib/nfagent/plugin.rb
+lib/nfagent/poller.rb
+lib/nfagent/server.rb
+lib/nfagent/submitter.rb
 lib/nfagent/tail.rb
 lib/nfagent/tests.rb
-lib/nfagent/submitter.rb
 bin/nfagent
 bin/squid_log_writer
-PostInstall.txt
-History.txt
-Manifest.txt
-README.txt
 nfagent.conf
 nfagent.init.ubuntu.txt
 nfagent.init.redhat.txt

data/PostInstall.txt

@@ -1 +1,7 @@
+
 For more information on nfagent, see http://nfagent.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/{README.txt → README.rdoc}

@@ -1,6 +1,6 @@
-= NetFox Agent
+= nfagent
 
-* http://netfox.com
+* http://github.com/#{github_username}/#{project_name}
 
 == DESCRIPTION:
 
@@ -8,33 +8,25 @@ Logging Agent for NetFox Online
 
 == FEATURES/PROBLEMS:
 
+* FIX (list of features or problems)
 
 == SYNOPSIS:
 
+  FIX (code sample of usage)
 
 == REQUIREMENTS:
 
-* Event Machine
-* SV Util
+* FIX (list of requirements)
 
 == INSTALL:
 
-sudo gem install nfagent
-
-== DEVELOPERS:
-
-After checking out the source, run:
-
-  $ rake newb
-
-This task will install any missing dependencies, run the tests/specs,
-and generate the RDoc.
+* FIX (sudo gem install, anything else)
 
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2010 FIX
+Copyright (c) 2011 FIXME full name
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -1,31 +1,28 @@
-%w[rubygems rake rake/clean hoe fileutils newgem rubigen].each { |f| require f }
-require File.dirname(__FILE__) + '/lib/nfagent'
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/nfagent'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
 
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
-$hoe = Hoe.spec('nfagent') do |p|
-  p.version = NFAgent::VERSION
-  p.summary = "Logging Agent for NetFox Online"
-  p.developer('Daniel Draper', 'daniel@netfox.com')
-  p.changes              = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  p.post_install_message = 'PostInstall.txt'
-  p.rubyforge_name       = p.name
-  p.extra_deps         = [
-    ['svutil','>= 0.0.12'], ['eventmachine', '>= 0.12.8']
-  ]
-  p.extra_dev_deps = [
-    ['newgem', ">= #{::Newgem::VERSION}"]
+$hoe = Hoe.spec 'nfagent' do
+  self.developer('Daniel Draper', 'daniel@netfox.com')
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps         = [
+    ['svutil','>= 0.1.3'], ['eventmachine', '>= 0.12.8'], ['squiggle', '>= 0.0.8']
   ]
-  p.clean_globs |= %w[**/.DS_Store tmp *.log]
-  path = (p.rubyforge_name == p.name) ? p.rubyforge_name : "\#{p.rubyforge_name}/\#{p.name}"
-  p.remote_rdoc_dir = File.join(path.gsub(/^#{p.rubyforge_name}\/?/,''), 'rdoc')
-  p.rsync_args = '-av --delete --ignore-errors'
-  p.readme_file = "README.txt"
-  p.spec_extras[:default_executable] = 'nfagent'
+
 end
 
-require 'newgem/tasks' # load /tasks/*.rake
+require 'newgem/tasks'
 Dir['tasks/**/*.rake'].each { |t| load t }
 
 # TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
 # task :default => [:spec, :features]

data/bin/squid_log_writer

@@ -6,12 +6,14 @@ class Client
   def initialize(host, port)
     @host = host
     @port = port
+    @count = 0
   end
 
   def write_safely(data)
     connect_socket unless @connected
     begin
       @client.puts(data)
+      sleep 0.01
     rescue
       @connected = false
     end

data/lib/nfagent.rb

@@ -2,7 +2,9 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 require 'rubygems'
+require 'active_support'
 require 'svutil'
+require 'squiggle'
 
 require 'fileutils'
 require 'logger'
@@ -13,10 +15,13 @@ require 'eventmachine'
 require 'em/timers'
 require 'rbconfig'
 
+require 'nfagent/object_extra'
 require 'nfagent/chunk'
 require 'nfagent/client'
 require 'nfagent/client_response'
 require 'nfagent/chunk_handler'
+require 'nfagent/mapper_proxy'
+require 'nfagent/plugin'
 require 'nfagent/submitter'
 require 'nfagent/encoder'
 require 'nfagent/config'
@@ -31,5 +36,5 @@ require 'nfagent/cli'
 require 'nfagent/tests'
 
 module NFAgent
-  VERSION = '0.9.31'
+  VERSION = '0.9.50'
 end

data/lib/nfagent/chunk.rb

@@ -2,44 +2,58 @@ require 'zlib'
 require 'digest'
 
 module NFAgent
-  class Chunk
+  class ChunkExpired < StandardError; end
+  class ChunkFull < StandardError; end
+  class DayBoundary < StandardError; end
+
+  class Chunk < Array
     attr_reader :created_at
+    attr_reader :max_size
 
-    ::DEFAULT_TIME_OUT = 60
+    DEFAULT_MAX_SIZE = 500
 
-    def initialize(max_size = 500)
+    def initialize(max_size = DEFAULT_MAX_SIZE)
       @max_size = max_size
       @created_at = Time.now
-      @array = []
-      @submitter = Submitter.new(Config.client_key)
     end
 
     def <<(line)
-      @array << line
+      raise ChunkExpired if expired?
+      raise ChunkFull if full?
+      raise DayBoundary if Time.now.day != self.created_at.day
+      super(line)
     end
 
     def full?
-      @array.size >= @max_size
+      self.size >= @max_size
     end
 
     def expired?
-      (Time.now - @created_at > ::DEFAULT_TIME_OUT) && !@array.empty?
+      (Time.now - @created_at > Config.chunk_timeout) && !self.empty?
     end
 
-    # TODO: Is this the right place for compression, encoding and check summing? Perhaps it should go into the submitter to that it can be deferred
-    def dump
+    def dump(key = nil)
       Payload.new do |payload|
-        Log.info("Dumping payload from chunk (#{@array.size} lines)")
-        payload.line_count = @array.size
+        Log.info("Dumping payload from chunk (#{self.size || 0} lines #{'due to expiry' if expired?}")
+        payload.line_count = self.size
         payload.chunk_expired = expired?
-        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
+        payload.key = key
+        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(self.join("\n"), Zlib::BEST_COMPRESSION))
         payload.checksum = Digest::SHA1.hexdigest(payload.data)
       end
     end
 
-    def clear
-      @array.clear
-      @created_at = Time.now
+    def submit(key = nil)
+      Log.info("Submitting...")
+      # TODO God knows why EM Deferrable isn't working - defer here is OK
+      EM.defer {
+        submitter = Submitter.new(self.dump(key))
+        submitter.errback { |payload|
+          payload.write_to_disk(Config.dump_dir)
+        }
+        submitter.perform
+      }
+      # Callback and remove from chunk group
     end
   end
 end

data/lib/nfagent/chunk_handler.rb

@@ -1,40 +1,72 @@
 module NFAgent
+  class LookUpError < StandardError; end
+  class IgnoreLine < StandardError; end
+
   class ChunkHandler
 
-    # TODO: Rename this to Controller later
-    def initialize(chunk_size = 500)
-      @chunk = Chunk.new(chunk_size)
+    attr_accessor :chunk_group
+
+    def initialize(options = {})
+      @chunk_size = options[:chunk_size] || 500
+      @parser = options[:parser] || Squiggle::SquidStandardParser.new(Config.time_zone)
+      @chunk_group = {}
+      class << @chunk_group
+        def fetch!(key, new_chunk)
+          if self.has_key?(key)
+            self.fetch(key)
+          else
+            self[key] = new_chunk
+            new_chunk
+          end
+        end
+      end
     end
 
     def append(line)
-      # if current day is > day of last entry on current_chunk 
-      # then submit and reset the chunk before adding the line
-      current_day = Time.now.day
-      if current_day != @chunk.created_at.day
-        Log.info("Expiring chunk due to date rollover")
-        reset_chunk
+      if Config.parse == 'locally'
+        parsed = @parser.parse(line)
+        return if parsed.invalid?
+        if Config.mode == 'multi'
+          begin
+            key = MapperProxy.find_account_id(parsed.username, parsed.client_ip)
+            # TODO: Still appending line as string until Server API has been updated
+            return append2(line, key)
+          rescue LookUpError, IgnoreLine
+            return # Do nothing
+          end
+        end
+      end
+      # TODO: rename append2
+      append2(line)
+    end
+
+    def append2(line, key = nil)
+      key ||= 'all'
+      begin
+        chunk = @chunk_group.fetch!(key, Chunk.new(@chunk_size))
+        chunk << line
+      rescue ChunkExpired, ChunkFull
+        Log.info("Chunk full or expired, cannot add lines")
+        reset_chunk(key)
       end
-      @chunk << line
     end
 
     def check_full_or_expired
-      if @chunk.full?
-        Log.info("Chunk Full: Resetting...")
-        reset_chunk
-      elsif @chunk.expired?
-        Log.info("Chunk Expired: Resetting...")
-        reset_chunk
+      @chunk_group.each_pair do |key, chunk|
+        if chunk.full?
+          Log.info("Chunk Full: Resetting...")
+          reset_chunk(key)
+        elsif chunk.expired?
+          Log.info("Chunk Expired: Resetting...")
+          reset_chunk(key)
+        end
       end
     end
 
     private
-      def reset_chunk
-        submitter = Submitter.new(@chunk.dump)
-        submitter.errback { |payload|
-          payload.write_to_disk(Config.dump_dir)
-        }
-        @chunk.clear
-        submitter.perform
+      def reset_chunk(key = nil)
+        chunk = @chunk_group.delete(key || 'all')
+        chunk.submit(key == 'all' ? nil : key)
       end
   end
 end

data/lib/nfagent/cli.rb

@@ -4,7 +4,7 @@ module NFAgent
     include SVUtil
 
     def initialize
-      Config.load_and_parse
+      Config.init
       if Config.test_mode?
         Tests.run
         exit 1

data/lib/nfagent/client.rb

@@ -1,6 +1,7 @@
 module NFAgent
   class Client
-     SERVICE_HOST = "collector.service.netfox.com"
+    # TODO: Make this a config option
+    SERVICE_HOST = "sandbox.netfox.com"
 
     def self.post(end_point, data_hash)
       proxy_class = Net::HTTP::Proxy(Config.http_proxy_host, Config.http_proxy_port, Config.http_proxy_user, Config.http_proxy_password)
@@ -8,14 +9,19 @@ module NFAgent
       proxy_class.start(SERVICE_HOST, 80) do |http|
         http.read_timeout = 120 # 2 minutes TODO: Make this a config option with 120 as default
         req = Net::HTTP::Post.new("/#{end_point}")
-        req.set_form_data(data_hash.merge("key" => Config.client_key))
+        p({"key" => Config.client_key}.merge(data_hash).delete('data'))
+        req.set_form_data({"key" => Config.client_key}.merge(data_hash))
         ClientResponse.new do |resp|
           resp.response, resp.message = http.request(req)
+          if !resp.ok?
+            Log.info("Client Returned with code (#{resp.response.code}, #{resp.response.msg}) and message '#{resp.message}'")
+          end
         end
       end
     rescue Exception => e
       # Trap Exception class here to ensure we catch Timeout
       ClientResponse.new do |resp|
+        Log.info("Client Error: #{$!}")
         resp.message = $!
       end
     end

data/lib/nfagent/config.rb

@@ -8,11 +8,41 @@ module NFAgent
       @@test_mode
     end
 
+    # Config Options
+    # client_key: String, the access key for the client (for the account in normal mode or for a partner in multi mode)
+    # dump_dir: String, the directory path of a local spool location
+    # pid_file: String, path of process ID file
+    # mode: (optional, default: 'normal') String, either 'normal' or 'multi' - can be left blank
+    # mapping: Class, this is a plugin class which must be stored in a file in the directory /etc/nfagent/plugins/
+    # parse: (optional, default: 'remotely'): String, either 'remotely' or 'locally'
+    #
+    defaults do |c|
+      c.mode = 'normal'
+      c.parse = 'remotely'
+      c.chunk_timeout = 60
+      c.time_zone = 'UTC'
+      c.plugin_directory = '/etc/nfagent/plugins/'
+    end
+
     class << self
       def validate
         unless dump_dir and File.exists?(dump_dir) and File.directory?(dump_dir)
           raise "Dump dir (#{dump_dir}) must exist and be a directory"
         end
+        # Mode
+        unless %w(normal multi).include?(mode)
+          raise "Invalid mode: must be one of 'normal' or 'multi'"
+        end
+        if mode == 'multi' && mapper.blank?
+          raise "Multi mode requires a mapper to be set"
+        end
+        if mode == 'multi' && parse != 'locally'
+          raise "Multi mode requires that parsing be done locally (set parse = 'locally')"
+        end
+        # Parse
+        unless %w(remotely locally).include?(parse)
+          raise "Invalid parse option: Must be one of 'remotely' or 'locally'"
+        end
         super
       end
 
@@ -30,6 +60,9 @@ module NFAgent
           opts.on("-T", "--test", "Run connection tests") do
             @@test_mode = true
           end
+          opts.on("-P", "--parse", "Parse locally before submitting") do
+            Config.parse_locally = true
+          end
         end
       end
     end