nfagent 0.9.30 → 0.9.31

data/History.txt

@@ -1,4 +1,4 @@
-=== 0.0.1 2011-02-03
+== 0.0.1 2009-09-22
 
 * 1 major enhancement:
   * Initial release

data/Manifest.txt

@@ -1,47 +1,29 @@
-History.txt
-Manifest.txt
-PostInstall.txt
-README.rdoc
 Rakefile
-script/console
-script/destroy
-script/generate
-test/test_chunk.rb
-test/test_chunk_handler.rb
-test/test_client.rb
-test/test_config.rb
-test/test_helper.rb
-test/test_mapper_proxy.rb
-test/test_nfagent.rb
-test/test_payload.rb
-test/test_plugin.rb
-test/config
-sandbox/dumps/debug
-test/plugins/my_mapper.rb
 lib/nfagent.rb
 lib/nfagent
-lib/nfagent/base64.rb
-lib/nfagent/chunk.rb
-lib/nfagent/chunk_handler.rb
 lib/nfagent/cli.rb
-lib/nfagent/client.rb
-lib/nfagent/client_response.rb
-lib/nfagent/config.rb
+lib/nfagent/base64.rb
 lib/nfagent/encoder.rb
-lib/nfagent/event.rb
-lib/nfagent/info.rb
+lib/nfagent/server.rb
+lib/nfagent/chunk_handler.rb
 lib/nfagent/log.rb
-lib/nfagent/mapper_proxy.rb
-lib/nfagent/object_extra.rb
+lib/nfagent/info.rb
+lib/nfagent/event.rb
 lib/nfagent/payload.rb
-lib/nfagent/plugin.rb
 lib/nfagent/poller.rb
-lib/nfagent/server.rb
-lib/nfagent/submitter.rb
+lib/nfagent/chunk.rb
+lib/nfagent/client.rb
+lib/nfagent/client_response.rb
+lib/nfagent/config.rb
 lib/nfagent/tail.rb
 lib/nfagent/tests.rb
+lib/nfagent/submitter.rb
 bin/nfagent
 bin/squid_log_writer
+PostInstall.txt
+History.txt
+Manifest.txt
+README.txt
 nfagent.conf
 nfagent.init.ubuntu.txt
 nfagent.init.redhat.txt

data/PostInstall.txt

@@ -1,7 +1 @@
-
 For more information on nfagent, see http://nfagent.rubyforge.org
-
-NOTE: Change this information in PostInstall.txt 
-You can also delete it if you don't want it.
-
-

data/{README.rdoc → README.txt}

@@ -1,6 +1,6 @@
-= nfagent
+= NetFox Agent
 
-* http://github.com/#{github_username}/#{project_name}
+* http://netfox.com
 
 == DESCRIPTION:
 
@@ -8,25 +8,33 @@ Logging Agent for NetFox Online
 
 == FEATURES/PROBLEMS:
 
-* FIX (list of features or problems)
 
 == SYNOPSIS:
 
-  FIX (code sample of usage)
 
 == REQUIREMENTS:
 
-* FIX (list of requirements)
+* Event Machine
+* SV Util
 
 == INSTALL:
 
-* FIX (sudo gem install, anything else)
+sudo gem install nfagent
+
+== DEVELOPERS:
+
+After checking out the source, run:
+
+  $ rake newb
+
+This task will install any missing dependencies, run the tests/specs,
+and generate the RDoc.
 
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2011 FIXME full name
+Copyright (c) 2010 FIX
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -1,28 +1,31 @@
-require 'rubygems'
-gem 'hoe', '>= 2.1.0'
-require 'hoe'
-require 'fileutils'
-require './lib/nfagent'
-
-Hoe.plugin :newgem
-# Hoe.plugin :website
-# Hoe.plugin :cucumberfeatures
+%w[rubygems rake rake/clean hoe fileutils newgem rubigen].each { |f| require f }
+require File.dirname(__FILE__) + '/lib/nfagent'
 
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
-$hoe = Hoe.spec 'nfagent' do
-  self.developer('Daniel Draper', 'daniel@netfox.com')
-  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
-  self.rubyforge_name       = self.name # TODO this is default value
-  self.extra_deps         = [
-    ['svutil','>= 0.1.3'], ['eventmachine', '>= 0.12.8'], ['squiggle', '>= 0.0.8']
+$hoe = Hoe.spec('nfagent') do |p|
+  p.version = NFAgent::VERSION
+  p.summary = "Logging Agent for NetFox Online"
+  p.developer('Daniel Draper', 'daniel@netfox.com')
+  p.changes              = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.post_install_message = 'PostInstall.txt'
+  p.rubyforge_name       = p.name
+  p.extra_deps         = [
+    ['svutil','>= 0.0.12'], ['eventmachine', '>= 0.12.8']
   ]
-
+  p.extra_dev_deps = [
+    ['newgem', ">= #{::Newgem::VERSION}"]
+  ]
+  p.clean_globs |= %w[**/.DS_Store tmp *.log]
+  path = (p.rubyforge_name == p.name) ? p.rubyforge_name : "\#{p.rubyforge_name}/\#{p.name}"
+  p.remote_rdoc_dir = File.join(path.gsub(/^#{p.rubyforge_name}\/?/,''), 'rdoc')
+  p.rsync_args = '-av --delete --ignore-errors'
+  p.readme_file = "README.txt"
+  p.spec_extras[:default_executable] = 'nfagent'
 end
 
-require 'newgem/tasks'
+require 'newgem/tasks' # load /tasks/*.rake
 Dir['tasks/**/*.rake'].each { |t| load t }
 
 # TODO - want other tests/tasks run by default? Add them to the list
-# remove_task :default
 # task :default => [:spec, :features]

data/bin/squid_log_writer

@@ -6,14 +6,12 @@ class Client
   def initialize(host, port)
     @host = host
     @port = port
-    @count = 0
   end
 
   def write_safely(data)
     connect_socket unless @connected
     begin
       @client.puts(data)
-      sleep 0.01
     rescue
       @connected = false
     end

data/lib/nfagent.rb

@@ -2,9 +2,7 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 require 'rubygems'
-require 'active_support'
 require 'svutil'
-require 'squiggle'
 
 require 'fileutils'
 require 'logger'
@@ -15,13 +13,10 @@ require 'eventmachine'
 require 'em/timers'
 require 'rbconfig'
 
-require 'nfagent/object_extra'
 require 'nfagent/chunk'
 require 'nfagent/client'
 require 'nfagent/client_response'
 require 'nfagent/chunk_handler'
-require 'nfagent/mapper_proxy'
-require 'nfagent/plugin'
 require 'nfagent/submitter'
 require 'nfagent/encoder'
 require 'nfagent/config'
@@ -36,5 +31,5 @@ require 'nfagent/cli'
 require 'nfagent/tests'
 
 module NFAgent
-  VERSION = '0.9.30'
+  VERSION = '0.9.31'
 end

data/lib/nfagent/chunk.rb

@@ -2,58 +2,44 @@ require 'zlib'
 require 'digest'
 
 module NFAgent
-  class ChunkExpired < StandardError; end
-  class ChunkFull < StandardError; end
-  class DayBoundary < StandardError; end
-
-  class Chunk < Array
+  class Chunk
     attr_reader :created_at
-    attr_reader :max_size
 
-    DEFAULT_MAX_SIZE = 500
+    ::DEFAULT_TIME_OUT = 60
 
-    def initialize(max_size = DEFAULT_MAX_SIZE)
+    def initialize(max_size = 500)
       @max_size = max_size
       @created_at = Time.now
+      @array = []
+      @submitter = Submitter.new(Config.client_key)
     end
 
     def <<(line)
-      raise ChunkExpired if expired?
-      raise ChunkFull if full?
-      raise DayBoundary if Time.now.day != self.created_at.day
-      super(line)
+      @array << line
     end
 
     def full?
-      self.size >= @max_size
+      @array.size >= @max_size
     end
 
     def expired?
-      (Time.now - @created_at > Config.chunk_timeout) && !self.empty?
+      (Time.now - @created_at > ::DEFAULT_TIME_OUT) && !@array.empty?
     end
 
-    def dump(key = nil)
+    # TODO: Is this the right place for compression, encoding and check summing? Perhaps it should go into the submitter to that it can be deferred
+    def dump
       Payload.new do |payload|
-        Log.info("Dumping payload from chunk (#{self.size || 0} lines #{'due to expiry' if expired?}")
-        payload.line_count = self.size
+        Log.info("Dumping payload from chunk (#{@array.size} lines)")
+        payload.line_count = @array.size
         payload.chunk_expired = expired?
-        payload.key = key
-        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(self.join("\n"), Zlib::BEST_COMPRESSION))
+        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
         payload.checksum = Digest::SHA1.hexdigest(payload.data)
       end
     end
 
-    def submit(key = nil)
-      Log.info("Submitting...")
-      # TODO God knows why EM Deferrable isn't working - defer here is OK
-      EM.defer {
-        submitter = Submitter.new(self.dump(key))
-        submitter.errback { |payload|
-          payload.write_to_disk(Config.dump_dir)
-        }
-        submitter.perform
-      }
-      # Callback and remove from chunk group
+    def clear
+      @array.clear
+      @created_at = Time.now
     end
   end
 end

data/lib/nfagent/chunk_handler.rb

@@ -1,72 +1,40 @@
 module NFAgent
-  class LookUpError < StandardError; end
-  class IgnoreLine < StandardError; end
-
   class ChunkHandler
 
-    attr_accessor :chunk_group
-
-    def initialize(options = {})
-      @chunk_size = options[:chunk_size] || 500
-      @parser = options[:parser] || Squiggle::SquidStandardParser.new(Config.time_zone)
-      @chunk_group = {}
-      class << @chunk_group
-        def fetch!(key, new_chunk)
-          if self.has_key?(key)
-            self.fetch(key)
-          else
-            self[key] = new_chunk
-            new_chunk
-          end
-        end
-      end
+    # TODO: Rename this to Controller later
+    def initialize(chunk_size = 500)
+      @chunk = Chunk.new(chunk_size)
     end
 
     def append(line)
-      if Config.parse == 'locally'
-        parsed = @parser.parse(line)
-        return if parsed.invalid?
-        if Config.mode == 'multi'
-          begin
-            key = MapperProxy.find_account_id(parsed.username, parsed.client_ip)
-            # TODO: Still appending line as string until Server API has been updated
-            return append2(line, key)
-          rescue LookUpError, IgnoreLine
-            return # Do nothing
-          end
-        end
-      end
-      # TODO: rename append2
-      append2(line)
-    end
-
-    def append2(line, key = nil)
-      key ||= 'all'
-      begin
-        chunk = @chunk_group.fetch!(key, Chunk.new(@chunk_size))
-        chunk << line
-      rescue ChunkExpired, ChunkFull
-        Log.info("Chunk full or expired, cannot add lines")
-        reset_chunk(key)
+      # if current day is > day of last entry on current_chunk 
+      # then submit and reset the chunk before adding the line
+      current_day = Time.now.day
+      if current_day != @chunk.created_at.day
+        Log.info("Expiring chunk due to date rollover")
+        reset_chunk
       end
+      @chunk << line
     end
 
     def check_full_or_expired
-      @chunk_group.each_pair do |key, chunk|
-        if chunk.full?
-          Log.info("Chunk Full: Resetting...")
-          reset_chunk(key)
-        elsif chunk.expired?
-          Log.info("Chunk Expired: Resetting...")
-          reset_chunk(key)
-        end
+      if @chunk.full?
+        Log.info("Chunk Full: Resetting...")
+        reset_chunk
+      elsif @chunk.expired?
+        Log.info("Chunk Expired: Resetting...")
+        reset_chunk
       end
     end
 
     private
-      def reset_chunk(key = nil)
-        chunk = @chunk_group.delete(key || 'all')
-        chunk.submit(key == 'all' ? nil : key)
+      def reset_chunk
+        submitter = Submitter.new(@chunk.dump)
+        submitter.errback { |payload|
+          payload.write_to_disk(Config.dump_dir)
+        }
+        @chunk.clear
+        submitter.perform
       end
   end
 end

data/lib/nfagent/cli.rb

@@ -4,12 +4,12 @@ module NFAgent
     include SVUtil
 
     def initialize
-      Config.init
+      Config.load_and_parse
       if Config.test_mode?
         Tests.run
         exit 1
       end
-      @process = ProcessManager.new(Server)
+      @process = ProcessManager.new(Server, Config)
       @process.start
     end
   end

data/lib/nfagent/client.rb

@@ -1,6 +1,6 @@
 module NFAgent
   class Client
-    SERVICE_HOST = "collector.service.netfox.com"
+     SERVICE_HOST = "collector.service.netfox.com"
 
     def self.post(end_point, data_hash)
       proxy_class = Net::HTTP::Proxy(Config.http_proxy_host, Config.http_proxy_port, Config.http_proxy_user, Config.http_proxy_password)
@@ -8,17 +8,14 @@ module NFAgent
       proxy_class.start(SERVICE_HOST, 80) do |http|
         http.read_timeout = 120 # 2 minutes TODO: Make this a config option with 120 as default
         req = Net::HTTP::Post.new("/#{end_point}")
-        p({"key" => Config.client_key}.merge(data_hash).delete('data'))
-        req.set_form_data({"key" => Config.client_key}.merge(data_hash))
+        req.set_form_data(data_hash.merge("key" => Config.client_key))
         ClientResponse.new do |resp|
           resp.response, resp.message = http.request(req)
-          Log.info("Client Returned with '#{resp.message}'")
         end
       end
     rescue Exception => e
       # Trap Exception class here to ensure we catch Timeout
       ClientResponse.new do |resp|
-        Log.info("Client Error: #{$!}")
         resp.message = $!
       end
     end

data/lib/nfagent/config.rb

@@ -8,41 +8,11 @@ module NFAgent
       @@test_mode
     end
 
-    # Config Options
-    # client_key: String, the access key for the client (for the account in normal mode or for a partner in multi mode)
-    # dump_dir: String, the directory path of a local spool location
-    # pid_file: String, path of process ID file
-    # mode: (optional, default: 'normal') String, either 'normal' or 'multi' - can be left blank
-    # mapping: Class, this is a plugin class which must be stored in a file in the directory /etc/nfagent/plugins/
-    # parse: (optional, default: 'remotely'): String, either 'remotely' or 'locally'
-    #
-    defaults do |c|
-      c.mode = 'normal'
-      c.parse = 'remotely'
-      c.chunk_timeout = 60
-      c.time_zone = 'UTC'
-      c.plugin_directory = '/etc/nfagent/plugins/'
-    end
-
     class << self
       def validate
         unless dump_dir and File.exists?(dump_dir) and File.directory?(dump_dir)
           raise "Dump dir (#{dump_dir}) must exist and be a directory"
         end
-        # Mode
-        unless %w(normal multi).include?(mode)
-          raise "Invalid mode: must be one of 'normal' or 'multi'"
-        end
-        if mode == 'multi' && mapper.blank?
-          raise "Multi mode requires a mapper to be set"
-        end
-        if mode == 'multi' && parse != 'locally'
-          raise "Multi mode requires that parsing be done locally (set parse = 'locally')"
-        end
-        # Parse
-        unless %w(remotely locally).include?(parse)
-          raise "Invalid parse option: Must be one of 'remotely' or 'locally'"
-        end
         super
       end
 
@@ -60,9 +30,6 @@ module NFAgent
           opts.on("-T", "--test", "Run connection tests") do
             @@test_mode = true
           end
-          opts.on("-P", "--parse", "Parse locally before submitting") do
-            Config.parse_locally = true
-          end
         end
       end
     end