nfagent 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2009-09-22
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,22 @@
+Rakefile
+lib/nfagent.rb
+lib/nfagent.init
+lib/nfagent
+lib/nfagent/cli.rb
+lib/nfagent/base64.rb
+lib/nfagent/encoder.rb
+lib/nfagent/server.rb
+lib/nfagent/chunk_handler.rb
+lib/nfagent/log.rb
+lib/nfagent/event.rb
+lib/nfagent/chunk.rb
+lib/nfagent/config.rb
+lib/nfagent/tail.rb
+lib/nfagent/submitter.rb
+bin/nfagent
+bin/squid_log_writer
+PostInstall.txt
+History.txt
+Manifest.txt
+nfagent.init
+nfagent.conf

data/PostInstall.txt

@@ -0,0 +1 @@
+For more information on nfagent, see http://nfagent.rubyforge.org

data/Rakefile

@@ -0,0 +1,31 @@
+%w[rubygems rake rake/clean hoe fileutils newgem rubigen].each { |f| require f }
+require File.dirname(__FILE__) + '/lib/nfagent'
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec('nfagent') do |p|
+  p.version = NFAgent::VERSION
+  p.summary = "Logging Agent for NetFox Online"
+  p.developer('Daniel Draper', 'daniel@netfox.com')
+  p.changes              = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.post_install_message = 'PostInstall.txt'
+  p.rubyforge_name       = p.name
+  p.extra_deps         = [
+    ['svutil','>= 0.0.3'],
+  ]
+  p.extra_dev_deps = [
+    ['newgem', ">= #{::Newgem::VERSION}"]
+  ]
+  p.clean_globs |= %w[**/.DS_Store tmp *.log]
+  path = (p.rubyforge_name == p.name) ? p.rubyforge_name : "\#{p.rubyforge_name}/\#{p.name}"
+  p.remote_rdoc_dir = File.join(path.gsub(/^#{p.rubyforge_name}\/?/,''), 'rdoc')
+  p.rsync_args = '-av --delete --ignore-errors'
+  p.readme_file = "README.txt"
+  p.spec_extras[:default_executable] = 'nfagent'
+end
+
+require 'newgem/tasks' # load /tasks/*.rake
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# task :default => [:spec, :features]

data/bin/nfagent

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'nfagent'
+NFAgent::CLI.new

data/bin/squid_log_writer

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+
+class Client
+  def initialize(host, port)
+    @host = host
+    @port = port
+  end
+
+  def write_safely(data)
+    connect_socket unless @connected
+    begin
+      @client.puts(data)
+    rescue
+      @connected = false
+    end
+  end
+
+  def close
+    @client.close if @client && @connected
+  end
+
+  private
+    def connect_socket
+      begin
+        @client = TCPSocket.new(@host, @port)
+        @connected = true
+      rescue
+        @connected = false
+      end
+    end
+end
+
+client = Client.new("127.0.0.1", "10000")
+
+while !$stdin.eof?
+  line = $stdin.readline
+  client.write_safely(line)
+  $stdout.flush
+end
+client.close

data/lib/nfagent.init

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# nfagent    This shell script takes care of starting and stopping
+#
+# chkconfig: 345 98 98
+# description: The NetFox logging agent
+
+start () {
+
+}

data/lib/nfagent.rb

@@ -0,0 +1,28 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'rubygems'
+require 'svutil'
+
+require 'thread'
+require 'fileutils'
+require 'logger'
+require 'pp'
+require 'uri'
+require 'net/http'
+require 'eventmachine'
+
+require 'nfagent/chunk'
+require 'nfagent/chunk_handler'
+require 'nfagent/submitter'
+require 'nfagent/encoder'
+require 'nfagent/config'
+require 'nfagent/log'
+require 'nfagent/tail'
+require 'nfagent/event'
+require 'nfagent/server'
+require 'nfagent/cli'
+
+module NFAgent
+  VERSION = '0.0.1'
+end

data/lib/nfagent/base64.rb

@@ -0,0 +1,7 @@
+module NFAgent
+  class Base64
+    def self.encode64url(str)
+      [str].pack('m').tr("+/","-_")
+    end
+  end
+end

data/lib/nfagent/chunk.rb

@@ -0,0 +1,46 @@
+require 'zlib'
+require 'digest'
+
+module NFAgent
+  class Chunk
+    attr_reader :created_at
+
+    ::DEFAULT_TIME_OUT = 60
+
+    def initialize(max_size = 500)
+      @max_size = max_size
+      @created_at = Time.now
+      @array = []
+      @submitter = Submitter.new(Config.client_key)
+    end
+
+    def <<(line)
+      @array << line
+    end
+
+    def full?
+      @array.size >= @max_size
+    end
+
+    def expired?
+      (Time.now - @created_at > ::DEFAULT_TIME_OUT) && !@array.empty?
+    end
+
+    def dump
+      puts @array.join("\n")
+      payload = Encoder.encode64url(Zlib::Deflate.deflate(@array.join("\n"), Zlib::BEST_COMPRESSION))
+      checksum = Digest::SHA1.hexdigest(payload)
+      Log.info "Submitting chunk to server (#{checksum})"
+      [ payload, checksum ]
+    end
+
+    def clear
+      @array.clear
+    end
+
+    def submit_to_server
+      payload, checksum = dump
+      @submitter.submit(payload, checksum)
+    end
+  end
+end

data/lib/nfagent/chunk_handler.rb

@@ -0,0 +1,57 @@
+module NFAgent
+  class ChunkHandler
+
+    def initialize(chunk_size = 500)
+      @mutex = Mutex.new
+      @chunk_size = chunk_size
+      make_new_chunk
+    end
+
+    def submit(line)
+      # if current day is > day of last entry on current_chunk 
+      # then submit and reset the chunk before adding the line
+      current_day = Time.now.day
+      if current_day != current_chunk.created_at.day
+        Log.info("Expiring chunk due to date rollover")
+        reset_chunk
+      end
+      current_chunk << line
+    end
+
+    def periodically_check_expired
+      Thread.new do
+        loop do
+          check_full_or_expired
+          sleep 5
+        end
+      end
+    end
+
+    private
+      def check_full_or_expired
+        if current_chunk.full? || current_chunk.expired?
+          reset_chunk
+        end
+      end
+
+      def reset_chunk
+        outgoing_chunk = current_chunk
+        make_new_chunk
+        Thread.new do
+          outgoing_chunk.submit_to_server
+        end
+      end
+
+      def make_new_chunk
+        @mutex.synchronize do
+          @current_chunk = Chunk.new(@chunk_size)
+        end
+      end
+  
+      def current_chunk
+        @mutex.synchronize do
+          return @current_chunk
+        end
+      end
+  end
+end

data/lib/nfagent/cli.rb

@@ -0,0 +1,14 @@
+module NFAgent
+  class CLI
+    include NFAgent
+    include SVUtil
+
+    def initialize
+      Config.load_and_parse
+      @process = ProcessManager.new(Server)
+      @process.start
+    end
+  end
+end
+
+# TODO Run EventMachine here later to allow clients to connect for real time log display

data/lib/nfagent/config.rb

@@ -0,0 +1,36 @@
+module NFAgent
+  class Proxy
+    attr_accessor :host, :port, :user, :password
+  end
+
+  class Config < SVUtil::Config
+    @@proxy = Proxy.new
+
+    def self.proxy
+      @@proxy
+    end
+
+    class << self
+      def validate
+        unless dump_dir and File.exists?(dump_dir) and File.directory?(dump_dir)
+          raise "Dump dir (#{dump_dir}) must exist and be a directory"
+        end
+        super
+      end
+
+      def process_options
+        parse_options do |opts|
+          opts.on("-k", "--client-key [key]", "Service client key") do |key|
+            Config.client_key = key
+            end
+          opts.on("-l", "--debug-log [log-file]", "Debug Log File") do |log|
+            Config.log_file = log
+          end
+          opts.on("-D", "--dump-dir [dir]", "Dump directory for failed chunks") do |dir|
+            Config.dump_dir = dir
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nfagent/encoder.rb

@@ -0,0 +1,7 @@
+module NFAgent
+  class Encoder
+    def self.encode64url(str)
+      [str].pack('m').tr("+/","-_")
+    end
+  end
+end

data/lib/nfagent/event.rb

@@ -0,0 +1,22 @@
+module NFAgent
+  class Event < EventMachine::Connection
+    def initialize(chunk_handler)
+      @handler = chunk_handler
+    end
+
+    def post_init
+      Log.info "Client Connected"
+    end
+
+    def receive_data(data)
+      if data && data.length > 2
+        @handler.submit(data)
+      end
+      send_data('OK')
+    end
+
+    def unbind
+      Log.info "Disconnected"
+    end
+  end
+end

data/lib/nfagent/log.rb

@@ -0,0 +1,15 @@
+module NFAgent
+  class Log
+    class << self
+      %w(info warning error).each do |level|
+        define_method(level) do |arg|
+          log(level, arg)
+        end
+      end
+
+      def log(level, arg)
+	      STDOUT.puts "#{Time.now}: (#{level}) #{arg}"
+      end
+    end
+  end
+end

data/lib/nfagent/server.rb

@@ -0,0 +1,18 @@
+module NFAgent
+  class Server
+    def initialize
+      Log.info("Starting up")
+      Submitter.start_resubmission_thread
+      chunk_handler = ChunkHandler.new
+      chunk_handler.periodically_check_expired
+
+      EM.run {
+        EM.start_server "0.0.0.0", 10000, Event, chunk_handler
+      }
+
+      #Tail.tail(Config.proxy_log) do |line|
+      #  chunk_handler.submit(line)
+      #end
+    end
+  end
+end

data/lib/nfagent/submitter.rb

@@ -0,0 +1,54 @@
+module NFAgent
+  class Submitter
+    attr_accessor :host
+
+    def initialize(key)
+      @service_host = "collector.service.netfox.com"
+      @key = key
+    end
+
+    def submit(payload, checksum)
+      puts "submitting paylod: #{payload}"
+      proxy_class = Net::HTTP::Proxy(Config.proxy.host, Config.proxy.port, Config.proxy.user, Config.proxy.password)
+      proxy_class.start(@service_host, 80) do |http|
+        req = Net::HTTP::Post.new('/collector')
+        req.set_form_data({ "payload" => payload, "checksum" => checksum, "key" => @key })
+        response, body = http.request(req)
+        raise body unless Net::HTTPOK === response
+      end
+    rescue
+      Log.error "Submission Failed: #{$!}"
+      write_failed_dump(payload, checksum)
+    end
+
+    def write_failed_dump(payload, checksum)
+      File.open(File.join(Config.dump_dir, checksum), "w") do |file|
+        file << payload
+      end
+    end
+
+    def self.start_resubmission_thread
+      Thread.new do
+        loop do
+          self.resubmit_failed_dumps
+          sleep 60
+        end
+      end
+    end
+
+    def self.resubmit_failed_dumps
+      submitter = Submitter.new(Config.client_key)
+      dump_dir = Dir.new(Config.dump_dir)
+      dump_dir.entries.select { |e| not e =~ /^\./ }.each do |entry|
+        Log.info "Resubmitting #{entry }"
+        payload = ""
+        ref = File.join(dump_dir.path, entry)
+        File.open(ref, "r") do |file|
+          payload << file.read
+        end
+        FileUtils.rm(ref)
+        submitter.submit(payload, entry)
+      end
+    end
+  end
+end

data/lib/nfagent/tail.rb

@@ -0,0 +1,129 @@
+##!/usr/bin/env ruby1.9
+
+# NOTE: There is a bug in Ruby 1.8.6p111 which prevents this from working properly - 1.9 is known to work
+
+module LogClient
+  class Tail
+    class BufferError < StandardError; end
+
+    ::BUFFER_SIZE = 1024
+    ::MAX_BUFFER_SIZE = 32768
+
+    def self.tail(filename)
+      loop do
+        Tail.new(filename).run do |line|
+          unless line.nil? || line.empty?
+            if block_given?
+              yield line
+            else
+              puts line
+            end
+          end
+        end
+      end
+    end
+  
+    def initialize(filename)
+      @buffer = ""
+      @filename = filename
+      reset_buffer_size
+      seek_and_setup
+    end
+  
+    # The real work
+    # Start at next start. Read until we get a new line and yield the line
+    # If we don't get a new line then just ignore and try again until we do
+    def run
+      begin
+        yield @first_line if block_given?
+        while (size = File::Stat.new(@filename).size) >= @next_start
+          size = @file.stat.size
+          reset_buffer_size
+          begin
+            line = ""
+            @file.seek(@next_start, File::SEEK_SET)
+            @file.read(@buf_size, @buffer)
+            buffer_start = @next_start
+            found_new_line = false
+            0.upto(@buffer.size - 1) do |index|
+              line << @buffer[index]
+              if @buffer[index].chr == "\n"
+                yield(line) if block_given?
+                line = ""
+                found_new_line = true
+                @next_start = buffer_start + index + 1
+              end
+            end
+            unless found_new_line || @buffer.empty?
+              raise BufferError
+            end
+          rescue BufferError
+            increment_buffer_size
+            retry
+          end
+	  sleep 0.01
+        end
+      rescue Errno::ENOENT
+        # Wait until the file is recreated
+        while !File.exists?(@filename)
+          sleep 0.05
+        end
+      end
+    end
+  
+    private
+      def seek_and_setup
+        @file = File.open(@filename, "r+")
+        size = @file.stat.size
+        pos = size
+        begin
+          if size >= @buf_size
+            @file.seek(size - @buf_size, File::SEEK_SET)
+          end
+          @buffer = @file.read
+          if @buffer.size > 0
+            index = @buffer.size - 1
+            # Find the last new line in the @buffer
+            # Make sure we got a new line somewhere
+            while (char = @buffer[index].chr) != "\n" && index >= 0
+              index -= 1
+            end
+            # Start reading from here next time
+            @next_start = pos - (@buf_size - index) + 1
+            @next_start = 0 if @next_start < 0
+            line = ""
+            if index > 1
+              index -= 1
+              # Go back to the previous new line (or the start of the @buffer)
+              while (char = @buffer[index].chr) != "\n"
+                line << char
+                index -= 1
+              end
+            end
+            # If index is 0 and the buffer is smaller than the file
+            # then we can try reading more chars until we find a new line
+            if index <= 0 and @buf_size < size
+              raise BufferError
+            end
+            @first_line = line.reverse
+          else
+            @next_start = pos
+          end
+        rescue BufferError
+          increment_buffer_size
+          retry
+        end
+      end
+
+      def increment_buffer_size
+        @buf_size = @buf_size * 2
+        if @buf_size > ::MAX_BUFFER_SIZE
+          raise BufferError, "Maximum buffer size exceeded"
+        end
+      end
+  
+      def reset_buffer_size
+        @buf_size = ::BUFFER_SIZE
+      end
+  end
+end

data/nfagent.conf

@@ -0,0 +1,5 @@
+client_key = 1234
+proxy_log = /tmp/access.log
+dump_dir = /tmp/dumps
+log_file = /tmp/debug
+pid_file = /tmp/nfagent.pid

data/nfagent.init

@@ -0,0 +1,49 @@
+#! /bin/sh
+
+### BEGIN INIT INFO
+# Provides:		nfagent
+# Required-Start:	$network $local_fs $remote_fs
+# Required-Stop:
+# Default-Start:	2 3 4 5
+# Default-Stop:		0 1 6
+# Short-Description:	NetFox Agent
+### END INIT INFO
+
+set -e
+. /lib/lsb/init-functions
+
+# /etc/init.d/pgship: start and stop the PGShip WAL Shipping Service
+
+PGSHIP_OPTS="-f /etc/nfagent.conf -d"
+
+# Are we running from init?
+run_by_init() {
+    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
+
+case "$1" in
+  start)
+	log_daemon_msg "NetFox Agent" "nfagent"
+	if start-stop-daemon --start --quiet --oknodo --pidfile /var/run/pgship.pid --exec /usr/sbin/pgship.bin -- $PGSHIP_OPTS; then
+	    log_end_msg 0
+	else
+	    log_end_msg 1
+	fi
+	;;
+  stop)
+	log_daemon_msg "NetFox Agent" "nfagent"
+	if start-stop-daemon --stop --quiet --oknodo --pidfile /var/run/pgship.pid; then
+	    log_end_msg 0
+	else
+	    log_end_msg 1
+	fi
+	;;
+
+  *)
+	log_action_msg "Usage: /etc/init.d/nfagent {start|stop}"
+	exit 1
+esac
+
+exit 0

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification 
+name: nfagent
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Daniel Draper
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-22 00:00:00 +09:30
+default_executable: nfagent
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: svutil
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.0.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: newgem
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+description: ""
+email: 
+- daniel@netfox.com
+executables: 
+- nfagent
+- squid_log_writer
+extensions: []
+
+extra_rdoc_files: 
+- PostInstall.txt
+- History.txt
+- Manifest.txt
+files: 
+- Rakefile
+- lib/nfagent.rb
+- lib/nfagent.init
+- lib/nfagent/cli.rb
+- lib/nfagent/base64.rb
+- lib/nfagent/encoder.rb
+- lib/nfagent/server.rb
+- lib/nfagent/chunk_handler.rb
+- lib/nfagent/log.rb
+- lib/nfagent/event.rb
+- lib/nfagent/chunk.rb
+- lib/nfagent/config.rb
+- lib/nfagent/tail.rb
+- lib/nfagent/submitter.rb
+- bin/nfagent
+- bin/squid_log_writer
+- PostInstall.txt
+- History.txt
+- Manifest.txt
+- nfagent.init
+- nfagent.conf
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: nfagent
+rubygems_version: 1.3.3
+signing_key: 
+specification_version: 3
+summary: Logging Agent for NetFox Online
+test_files: []
+