nfagent 0.9.0 → 0.9.5

data/Manifest.txt

@@ -7,8 +7,13 @@ lib/nfagent/encoder.rb
 lib/nfagent/server.rb
 lib/nfagent/chunk_handler.rb
 lib/nfagent/log.rb
+lib/nfagent/info.rb
 lib/nfagent/event.rb
+lib/nfagent/payload.rb
+lib/nfagent/poller.rb
 lib/nfagent/chunk.rb
+lib/nfagent/client.rb
+lib/nfagent/client_response.rb
 lib/nfagent/config.rb
 lib/nfagent/tail.rb
 lib/nfagent/submitter.rb

data/Rakefile

@@ -11,7 +11,7 @@ $hoe = Hoe.spec('nfagent') do |p|
   p.post_install_message = 'PostInstall.txt'
   p.rubyforge_name       = p.name
   p.extra_deps         = [
-    ['svutil','>= 0.0.3'], ['eventmachine', '>= 0.12.8']
+    ['svutil','>= 0.0.6'], ['eventmachine', '>= 0.12.8']
   ]
   p.extra_dev_deps = [
     ['newgem', ">= #{::Newgem::VERSION}"]

data/lib/nfagent.rb

@@ -4,25 +4,31 @@ $:.unshift(File.dirname(__FILE__)) unless
 require 'rubygems'
 require 'svutil'
 
-require 'thread'
 require 'fileutils'
 require 'logger'
 require 'pp'
 require 'uri'
 require 'net/http'
 require 'eventmachine'
+require 'em/timers'
+require 'rbconfig'
 
 require 'nfagent/chunk'
+require 'nfagent/client'
+require 'nfagent/client_response'
 require 'nfagent/chunk_handler'
 require 'nfagent/submitter'
 require 'nfagent/encoder'
 require 'nfagent/config'
 require 'nfagent/log'
+require 'nfagent/info'
+require 'nfagent/payload'
+require 'nfagent/poller'
 require 'nfagent/tail'
 require 'nfagent/event'
 require 'nfagent/server'
 require 'nfagent/cli'
 
 module NFAgent
-  VERSION = '0.9.0'
+  VERSION = '0.9.5'
 end

data/lib/nfagent/chunk.rb

@@ -26,19 +26,16 @@ module NFAgent
       (Time.now - @created_at > ::DEFAULT_TIME_OUT) && !@array.empty?
     end
 
+    # TODO: Is this the right place for compression, encoding and check summing? Perhaps it should go into the submitter to that it can be deferred
     def dump
-      payload = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
-      checksum = Digest::SHA1.hexdigest(payload)
-      [ payload, checksum ]
+      Payload.new do |payload|
+        payload.data = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
+        payload.checksum = Digest::SHA1.hexdigest(payload.data)
+      end
     end
 
     def clear
       @array.clear
     end
-
-    def submit_to_server
-      payload, checksum = dump
-      @submitter.submit(payload, checksum)
-    end
   end
 end

data/lib/nfagent/chunk_handler.rb

@@ -1,57 +1,36 @@
 module NFAgent
   class ChunkHandler
 
+    # TODO: Rename this to Controller later
     def initialize(chunk_size = 500)
-      @mutex = Mutex.new
-      @chunk_size = chunk_size
-      make_new_chunk
+      @chunk = Chunk.new(chunk_size)
     end
 
-    def submit(line)
+    def append(line)
       # if current day is > day of last entry on current_chunk 
       # then submit and reset the chunk before adding the line
       current_day = Time.now.day
-      if current_day != current_chunk.created_at.day
+      if current_day != @chunk.created_at.day
         Log.info("Expiring chunk due to date rollover")
         reset_chunk
       end
-      current_chunk << line
+      @chunk << line
     end
 
-    def periodically_check_expired
-      Thread.new do
-        loop do
-          check_full_or_expired
-          sleep 5
-        end
+    def check_full_or_expired
+      if @chunk.full? || @chunk.expired?
+        reset_chunk
       end
     end
 
     private
-      def check_full_or_expired
-        if current_chunk.full? || current_chunk.expired?
-          reset_chunk
-        end
-      end
-
       def reset_chunk
-        outgoing_chunk = current_chunk
-        make_new_chunk
-        Thread.new do
-          outgoing_chunk.submit_to_server
-        end
-      end
-
-      def make_new_chunk
-        @mutex.synchronize do
-          @current_chunk = Chunk.new(@chunk_size)
-        end
-      end
-  
-      def current_chunk
-        @mutex.synchronize do
-          return @current_chunk
-        end
+        submitter = Submitter.new(@chunk.dump)
+        submitter.errback { |payload|
+          payload.write_to_disk(Config.dump_dir)
+        }
+        @chunk.clear
+        submitter.perform
       end
   end
 end

data/lib/nfagent/client.rb

@@ -0,0 +1,21 @@
+module NFAgent
+  class Client
+     SERVICE_HOST = "collector.service.netfox.com"
+
+    def self.post(end_point, data_hash)
+      proxy_class = Net::HTTP::Proxy(Config.proxy.host, Config.proxy.port, Config.proxy.user, Config.proxy.password)
+      # TODO: Enable SSL
+      proxy_class.start(SERVICE_HOST, 80) do |http|
+        req = Net::HTTP::Post.new("/#{end_point}")
+        req.set_form_data(data_hash.merge("key" => Config.client_key))
+        ClientResponse.new do |resp|
+          resp.response, resp.message = http.request(req)
+        end
+      end
+    rescue
+      ClientResponse.new do |resp|
+        resp.message = $!
+      end
+    end
+  end
+end

data/lib/nfagent/client_response.rb

@@ -0,0 +1,13 @@
+module NFAgent
+  class ClientResponse
+    attr_accessor :response, :message
+
+    def initialize
+      yield self if block_given?
+    end
+
+    def ok?
+      Net::HTTPOK === response
+    end
+  end
+end

data/lib/nfagent/config.rb

@@ -3,6 +3,8 @@ module NFAgent
     attr_accessor :host, :port, :user, :password
   end
 
+  RBConfig = Config
+
   class Config < SVUtil::Config
     @@proxy = Proxy.new
 

data/lib/nfagent/event.rb

@@ -1,7 +1,8 @@
 module NFAgent
   class Event < EventMachine::Connection
-    def initialize(chunk_handler)
+    def initialize(chunk_handler, poller)
       @handler = chunk_handler
+      @poller = poller
     end
 
     def post_init
@@ -10,7 +11,7 @@ module NFAgent
 
     def receive_data(data)
       if data && data.length > 2
-        @handler.submit(data)
+        @handler.append(data)
       end
       send_data('OK')
     end

data/lib/nfagent/info.rb

@@ -0,0 +1,26 @@
+module NFAgent
+  class Info
+    attr_accessor :last_proxy_connection
+
+    def hostname
+      Socket.gethostname
+    end
+
+    def version
+      NFAgent::VERSION
+    end
+
+    def host_string
+      RBConfig::CONFIG['host']
+    end
+
+    def to_hash
+      {
+        :hostname => hostname,
+        :version => version,
+        :host_string => host_string,
+        :last_proxy_connection => last_proxy_connection
+      }
+    end
+  end
+end

data/lib/nfagent/payload.rb

@@ -0,0 +1,70 @@
+
+module NFAgent
+  class Payload < Struct.new(:data, :checksum, :filename)
+    def initialize
+      yield self
+    end
+
+    def attempt
+      @attempt || 1
+    end
+
+    def attempt=(value)
+      @attempt = value.to_i
+    end
+
+    def increment_attempt!
+      @attempt ||= 1
+      @attempt += 1
+    end
+
+    def size
+      (self.data || "").size + 1
+    end
+
+    def write_to_disk(directory)
+      File.open(File.join(directory, "#{self.checksum}-#{self.attempt}"), "w") do |file|
+        file << self.data
+      end
+    end
+
+    def lock
+      return if locked?
+      FileUtils.touch(lockfile) if filename
+      yield
+      FileUtils.rm_f(lockfile) if filename
+    end
+
+    def locked?
+      filename && File.exists?(lockfile)
+    end
+
+    def self.read_from_file(filename)
+      # Ensure the file is only relative
+      filename = File.basename(filename)
+      self.new do |payload|
+        payload.filename = filename
+        payload.checksum, payload.attempt = filename.split("-")
+        payload.data = ""
+        ref = File.join(Config.dump_dir, filename)
+        File.open(ref, "r") do |file|
+          payload.data << file.read
+        end
+      end
+    end
+
+    def destroy!
+      FileUtils.rm_f(File.join(Config.dump_dir, self.filename)) if self.filename
+    end
+
+    def try_again_later
+      # TODO: Move the file to a new name with a later timetamp
+      FileUtils.mv(File.join(Config.dump_dir, self.filename), File.join(Config.dump_dir, "#{self.checksum}-#{self.attempt}"))
+    end
+
+    private
+      def lockfile
+        File.join(Config.dump_dir, "#{filename}.lock") if filename
+      end
+  end
+end

data/lib/nfagent/poller.rb

@@ -0,0 +1,18 @@
+module NFAgent
+  class Poller
+    include EM::Deferrable
+
+    def initialize
+      @info = Info.new
+    end
+
+    def send_heartbeat
+      payload = @info.to_hash
+      Log.info("Polling: #{payload.inspect}")
+      response = Client.post(:poller, payload)
+      if !response.ok?
+        Log.error("Poll Failed: #{response.message}")
+      end
+    end
+  end
+end

data/lib/nfagent/server.rb

@@ -1,14 +1,26 @@
 module NFAgent
   class Server
-    def initialize
+    def run
       Log.info("Starting up")
-      Submitter.start_resubmission_thread
       chunk_handler = ChunkHandler.new
-      chunk_handler.periodically_check_expired
+      poller = Poller.new
 
       EM.run {
-        EM.start_server "0.0.0.0", 10000, Event, chunk_handler
+        EM::PeriodicTimer.new(5) do
+          chunk_handler.check_full_or_expired
+        end
+        EM::PeriodicTimer.new(60) do
+          Submitter.resubmit_failed_dumps
+        end
+        EM::PeriodicTimer.new(120) do
+          poller.send_heartbeat
+        end
+        EM.start_server "0.0.0.0", 10000, Event, chunk_handler, poller
       }
     end
+
+    def shutdown
+      EM::stop_event_loop
+    end
   end
 end

data/lib/nfagent/submitter.rb

@@ -1,59 +1,47 @@
 module NFAgent
   class Submitter
+    include EM::Deferrable
     attr_accessor :host
 
-    def initialize(key)
-      @service_host = "collector.service.netfox.com"
-      @key = key
+    def initialize(payload)
+      @payload = payload
     end
 
-    def submit(payload, checksum, options = {})
-      attempt = options[:attempt] || 1
-      raise "Delaying attempt" unless [ 1, 2, 4, 8, 16 ].include?(attempt)
-      puts "Submitting Payload: #{checksum}, Attempt #{attempt}, (#{payload.size + 1} bytes)"
-      proxy_class = Net::HTTP::Proxy(Config.proxy.host, Config.proxy.port, Config.proxy.user, Config.proxy.password)
-      proxy_class.start(@service_host, 80) do |http|
-        req = Net::HTTP::Post.new('/collector')
-        req.set_form_data({ "payload" => payload, "checksum" => checksum, "key" => @key })
-        response, body = http.request(req)
-        raise body unless Net::HTTPOK === response
-      end
-    rescue
-      Log.error "Submission Failed: #{$!}"
-      write_failed_dump(payload, checksum, attempt)
-    end
-
-    def write_failed_dump(payload, checksum, attempt)
-      File.open(File.join(Config.dump_dir, "#{checksum}-#{attempt}"), "w") do |file|
-        file << payload
-      end
-    end
-
-    def self.start_resubmission_thread
-      Thread.new do
-        loop do
-          self.resubmit_failed_dumps
-          sleep 60
+    def perform
+      @payload.increment_attempt!
+      fail(@payload) unless [ 1, 2, 4, 8, 16 ].include?(@payload.attempt)
+      Log.info "Submitting Payload: #{@payload.checksum}, Attempt #{@payload.attempt}, (#{@payload.size} bytes)"
+      @payload.lock do
+        response = Client.post(:collector, "payload" => @payload.data, "checksum" => @payload.checksum)
+        if response.ok?
+          succeed(@payload)
+        else
+          Log.error "Submission Failed: #{response.message}"
+          fail(@payload)
         end
       end
     end
 
+    # TODO: Change attempt logic
+    # Add the next timestamp for when submission should be attenpted again to the end of the filename 
     def self.resubmit_failed_dumps
       submitter = Submitter.new(Config.client_key)
       dump_dir = Dir.new(Config.dump_dir)
       dump_dir.entries.select { |e| not e =~ /^\./ }.each do |entry|
-        dump_file, attempt = entry.split("-")
         Log.info "Resubmitting #{entry}"
-        payload = ""
-        ref = File.join(dump_dir.path, entry)
-        File.open(ref, "r") do |file|
-          payload << file.read
-        end
-        FileUtils.rm_f(ref)
-        attempt = attempt.to_i + 1
-        if attempt <= 16
-          submitter.submit(payload, dump_file, :attempt => attempt)
-        end
+        payload = Payload.read_from_file(entry)
+        submitter = self.new(payload)
+        submitter.callback { |payload|
+          payload.destroy!
+        }
+        submitter.errback { |payload|
+          if payload.attempt > 16
+            payload.destroy!
+          else
+            payload.try_again_later
+          end
+        }
+        submitter.perform
       end
     end
   end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 0
-  version: 0.9.0
+  - 5
+  version: 0.9.5
 platform: ruby
 authors: 
 - Daniel Draper
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-07 00:00:00 +09:30
+date: 2010-07-12 00:00:00 +09:30
 default_executable: nfagent
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,8 +27,8 @@ dependencies:
         segments: 
         - 0
         - 0
-        - 3
-        version: 0.0.3
+        - 6
+        version: 0.0.6
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -96,8 +96,13 @@ files:
 - lib/nfagent/server.rb
 - lib/nfagent/chunk_handler.rb
 - lib/nfagent/log.rb
+- lib/nfagent/info.rb
 - lib/nfagent/event.rb
+- lib/nfagent/payload.rb
+- lib/nfagent/poller.rb
 - lib/nfagent/chunk.rb
+- lib/nfagent/client.rb
+- lib/nfagent/client_response.rb
 - lib/nfagent/config.rb
 - lib/nfagent/tail.rb
 - lib/nfagent/submitter.rb