nfagent 0.9.27 → 0.9.28

data/Rakefile

@@ -15,7 +15,7 @@ $hoe = Hoe.spec 'nfagent' do
   self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
   self.extra_deps         = [
-    ['svutil','>= 0.1.2'], ['eventmachine', '>= 0.12.8'], ['squiggle', '>= 0.0.8']
+    ['svutil','>= 0.1.3'], ['eventmachine', '>= 0.12.8'], ['squiggle', '>= 0.0.8']
   ]
 
 end

data/bin/squid_log_writer

@@ -6,12 +6,14 @@ class Client
   def initialize(host, port)
     @host = host
     @port = port
+    @count = 0
   end
 
   def write_safely(data)
     connect_socket unless @connected
     begin
       @client.puts(data)
+      sleep 0.01
     rescue
       @connected = false
     end

data/lib/nfagent.rb

@@ -36,5 +36,5 @@ require 'nfagent/cli'
 require 'nfagent/tests'
 
 module NFAgent
-  VERSION = '0.9.27'
+  VERSION = '0.9.28'
 end

data/lib/nfagent/chunk.rb

@@ -34,7 +34,7 @@ module NFAgent
 
     def dump(key = nil)
       Payload.new do |payload|
-        Log.info("Dumping payload from chunk (#{self.size} lines)")
+        Log.info("Dumping payload from chunk (#{self.size || 0} lines #{'due to expiry' if expired?}")
         payload.line_count = self.size
         payload.chunk_expired = expired?
         payload.key = key
@@ -44,11 +44,15 @@ module NFAgent
     end
 
     def submit(key = nil)
-      submitter = Submitter.new(self.dump(key))
-      submitter.errback { |payload|
-        payload.write_to_disk(Config.dump_dir)
+      Log.info("Submitting...")
+      # TODO God knows why EM Deferrable isn't working - defer here is OK
+      EM.defer {
+        submitter = Submitter.new(self.dump(key))
+        submitter.errback { |payload|
+          payload.write_to_disk(Config.dump_dir)
+        }
+        submitter.perform
       }
-      submitter.perform
       # Callback and remove from chunk group
     end
   end

data/lib/nfagent/chunk_handler.rb

@@ -39,6 +39,7 @@ module NFAgent
         chunk = @chunk_group.fetch!(key, Chunk.new(@chunk_size))
         chunk << line
       rescue ChunkExpired, ChunkFull
+        Log.info("Chunk full or expired, cannot add lines")
         reset_chunk(key)
       end
     end
@@ -57,9 +58,8 @@ module NFAgent
 
     private
       def reset_chunk(key = nil)
-        key ||= 'all'
-        chunk = @chunk_group.delete(key)
-        chunk.submit
+        chunk = @chunk_group.delete(key || 'all')
+        chunk.submit(key == 'all' ? nil : key)
       end
   end
 end

data/lib/nfagent/client.rb

@@ -8,14 +8,17 @@ module NFAgent
       proxy_class.start(SERVICE_HOST, 80) do |http|
         http.read_timeout = 120 # 2 minutes TODO: Make this a config option with 120 as default
         req = Net::HTTP::Post.new("/#{end_point}")
+        p({"key" => Config.client_key}.merge(data_hash).delete('data'))
         req.set_form_data({"key" => Config.client_key}.merge(data_hash))
         ClientResponse.new do |resp|
           resp.response, resp.message = http.request(req)
+          Log.info("Client Returned with '#{resp.message}'")
         end
       end
     rescue Exception => e
       # Trap Exception class here to ensure we catch Timeout
       ClientResponse.new do |resp|
+        Log.info("Client Error: #{$!}")
         resp.message = $!
       end
     end

data/lib/nfagent/event.rb

@@ -13,7 +13,7 @@ module NFAgent
       if data && data.length > 10
         @handler.append(data)
       end
-      send_data('OK')
+      #send_data('OK')
     end
 
     def unbind

data/lib/nfagent/payload.rb

@@ -34,7 +34,8 @@ module NFAgent
     end
 
     def write_to_disk(directory)
-      File.open(File.join(directory, "#{self.checksum}-#{self.attempt}"), "w") do |file|
+      filename = [ self.checksum, self.attempt, self.key ].compact.join("-")
+      File.open(File.join(directory, filename), "w") do |file|
         file << self.data
       end
     end
@@ -50,14 +51,14 @@ module NFAgent
       filename && File.exists?(lockfile)
     end
 
-    def self.read_from_file(filename)
+    def self.read_from_file(filename, dir = Config.dump_dir)
       # Ensure the file is only relative
       filename = File.basename(filename)
       self.new do |payload|
         payload.filename = filename
-        payload.checksum, payload.attempt = filename.split("-")
+        payload.checksum, payload.attempt, payload.key = filename.split("-")
         payload.data = ""
-        ref = File.join(Config.dump_dir, filename)
+        ref = File.join(dir, filename)
         File.open(ref, "r") do |file|
           payload.data << file.read
         end
@@ -70,7 +71,8 @@ module NFAgent
 
     def try_again_later
       # TODO: Move the file to a new name with a later timetamp
-      FileUtils.mv(File.join(Config.dump_dir, self.filename), File.join(Config.dump_dir, "#{self.checksum}-#{self.attempt}"))
+      new_filename = [ self.checksum, self.attempt, self.key ].compact.join("-")
+      FileUtils.mv(File.join(Config.dump_dir, self.filename), File.join(Config.dump_dir, new_filename))
     end
 
     private

data/lib/nfagent/submitter.rb

@@ -15,6 +15,7 @@ module NFAgent
         response = Client.post(:collector, @payload.to_hash)
         if response.ok?
           succeed(@payload)
+          Log.info("Submitted #{@payload.line_count} lines")
         else
           Log.error "Submission Failed: #{response.message}"
           fail(@payload)

data/nfagent.conf

@@ -1,4 +1,4 @@
-client_key = 1234
-dump_dir = /tmp/dumps
-log_file = /tmp/debug
-pid_file = /tmp/nfagent.pid
+client_key = FNX3tyOkG6J7nk2uC3QX
+dump_dir = sandbox/dumps
+log_file = sandbox/debug
+pid_file = sandbox/nfagent.pid

data/test/test_chunk.rb

@@ -56,7 +56,7 @@ class TestChunk < ActiveSupport::TestCase
   end
 
   test "submit" do
-    NFAgent::Submitter.any_instance.expects(:perform)
+    EM.expects(:defer)
     chunk = NFAgent::Chunk.new
     chunk.submit
   end

data/test/test_chunk_handler.rb

@@ -41,11 +41,11 @@ class TestChunkHandler < ActiveSupport::TestCase
     assert_equal 1, chunk_handler.chunk_group['jetson'].size
   end
 
-  test "reset chunk after expiry" do
+  test "reset chunk after expiry in multi mode" do
     NFAgent::Config.parse = 'locally'
     NFAgent::Config.mode = 'multi'
     NFAgent::Config.mapper = 'MyMapper'
-    NFAgent::Submitter.any_instance.expects(:perform).times(2)
+    NFAgent::Chunk.any_instance.expects(:submit).at_least_once
     chunk_handler = NFAgent::ChunkHandler.new
     chunk_handler.append(@logline)
     Timecop.travel(30) do
@@ -67,11 +67,11 @@ class TestChunkHandler < ActiveSupport::TestCase
     end
   end
 
-  test "reset chunk after full wuth check_full_or_expired" do
+  test "reset chunk after full with check_full_or_expired in multi mode" do
     NFAgent::Config.parse = 'locally'
     NFAgent::Config.mode = 'multi'
     NFAgent::Config.mapper = 'MyMapper'
-    NFAgent::Submitter.any_instance.expects(:perform).times(2)
+    EM.expects(:defer).times(2)
     chunk_handler = NFAgent::ChunkHandler.new(:chunk_size => 10)
     9.times { chunk_handler.append(@logline) }
     9.times { chunk_handler.append(@logline2) }
@@ -91,4 +91,28 @@ class TestChunkHandler < ActiveSupport::TestCase
     chunk_handler.append("") # Invalid logline
     assert chunk_handler.chunk_group['all'].nil?
   end
+
+  test "reset chunk is passed appropriate key in multi mode" do
+    NFAgent::Config.parse = 'locally'
+    NFAgent::Config.mode = 'multi'
+    NFAgent::Config.mapper = 'MyMapper'
+    NFAgent::Chunk.any_instance.expects(:submit).with('acme')
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(61) do
+      chunk_handler.check_full_or_expired
+    end
+  end
+
+  test "reset chunk is passed nil key in normal mode" do
+    NFAgent::Config.parse = 'remotely'
+    NFAgent::Config.mode = 'normal'
+    NFAgent::Chunk.any_instance.expects(:submit).with(nil)
+    chunk_handler = NFAgent::ChunkHandler.new
+    chunk_handler.append(@logline)
+    Timecop.travel(61) do
+      chunk_handler.check_full_or_expired
+    end
+  end
+
 end

data/test/test_payload.rb

@@ -3,6 +3,7 @@ require 'test/test_helper'
 
 class TestPayload < ActiveSupport::TestCase
   setup do
+    FileUtils.rm_f(Dir.glob("test/sandbox/dumps/*"))
   end
 
   test "to_hash" do
@@ -23,4 +24,54 @@ class TestPayload < ActiveSupport::TestCase
     end
     assert_equal({ 'payload' => 'data', 'checksum' => 'checksum', 'line_count' => 10, 'chunk_expired' => false, 'key' => '1234' }, payload.to_hash)
   end
+
+  test "write-to-disk" do
+    payload = NFAgent::Payload.new do |p|
+      p.data = "data"
+      p.checksum = "checksum"
+      p.line_count = 10
+      p.chunk_expired = false
+    end
+    payload.write_to_disk("test/sandbox/dumps")
+    assert File.exists?("test/sandbox/dumps/checksum-0")
+    # And read back in
+    payload = NFAgent::Payload.read_from_file("checksum-0", "test/sandbox/dumps/")
+    assert_equal "data", payload.data
+    assert_equal 0, payload.attempt
+    assert !payload.key
+  end
+
+  test "write-to-disk with non-zero attempt number" do
+    payload = NFAgent::Payload.new do |p|
+      p.data = "data"
+      p.checksum = "checksum"
+      p.line_count = 10
+      p.chunk_expired = false
+      p.attempt = 10
+    end
+    payload.write_to_disk("test/sandbox/dumps")
+    assert File.exists?("test/sandbox/dumps/checksum-10")
+    # And read back in
+    payload = NFAgent::Payload.read_from_file("checksum-10", "test/sandbox/dumps/")
+    assert_equal "data", payload.data
+    assert_equal 10, payload.attempt
+  end
+
+  test "write-to-disk with key" do
+    payload = NFAgent::Payload.new do |p|
+      p.data = "data"
+      p.checksum = "checksum"
+      p.line_count = 10
+      p.chunk_expired = false
+      p.key = '1234'
+    end
+    payload.write_to_disk("test/sandbox/dumps")
+    assert File.exists?("test/sandbox/dumps/checksum-0-1234")
+    # And read back in
+    payload = NFAgent::Payload.read_from_file("checksum-0-1234", "test/sandbox/dumps/")
+    assert_equal "data", payload.data
+    assert_equal 0, payload.attempt
+    assert_equal '1234', payload.key
+  end
+
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 27
-  version: 0.9.27
+  - 28
+  version: 0.9.28
 platform: ruby
 authors: 
 - Daniel Draper
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-08 00:00:00 +10:30
+date: 2011-02-09 00:00:00 +10:30
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,8 +27,8 @@ dependencies:
         segments: 
         - 0
         - 1
-        - 2
-        version: 0.1.2
+        - 3
+        version: 0.1.3
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -60,7 +60,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: rubyforge
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     requirements: 
@@ -68,11 +68,25 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 2
-        - 4
         - 0
-        version: 2.4.0
+        - 4
+        version: 2.0.4
   type: :development
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 6
+        - 0
+        version: 2.6.0
+  type: :development
+  version_requirements: *id005
 description: Logging Agent for NetFox Online
 email: 
 - daniel@netfox.com