panmind-usage-tracker-server 1.1.5

data/README.md

@@ -0,0 +1,39 @@
+Panmind Usage Tracker Server
+----------------------------
+
+What is it?
+===========
+
+An `EventMachine`-based server that opens an UDP socket and sends out received data to a database
+
+Does it work?
+=============
+
+Yes, we are using it in production.
+
+Deploying
+=========
+
+ * Install the gem on the target machine and run it with this command:
+
+     $ usage_tracker [environment]
+
+   If you run it into a Rails.root it will log and write pids in canonical dirs.
+
+   or can be put under Upstart using the provided configuration file located in
+   `config/usage_tracker_server_upstart.conf`. Check it out and modify it to suit your needs.
+
+   The daemon logs to `usage_tracker_server.log` if the log directory exists and rotates its
+   logs when receives the USR1 signal.
+
+ * The daemon writes its pid into usage\_tracker\_server.pid
+
+ * The daemon can be configured to work with couchdb or mongodb adapter. Look at the
+   sample configuration file for hints.
+
+ * If the daemon cannot start, e.g. because of unavailable database or listening
+   address, it will print a diagnostic message to STDERR, log to usage\_tracker.log
+   and exit with status of 1.
+
+ * The daemon exits gracefully if it receives the INT or the TERM signals.
+

data/Rakefile

@@ -0,0 +1,35 @@
+# encoding: utf-8
+#
+require 'rake'
+require 'rdoc/task'
+
+begin
+  require 'jeweler'
+
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name        = 'panmind-usage-tracker-server'
+    gemspec.summary     = 'server component for usage-tracker-middleware (Rack)'
+    gemspec.description = 'this software implements an EventMachine reactor ' \
+                          'which stores json-objects received via UDP in a database.'
+
+    gemspec.authors     = ['Christian Woerner', 'Fabrizio Regini', 'Marcello Barnaba']
+    gemspec.homepage    = 'http://github.com/Panmind/usage_tracker_server'
+    gemspec.email       = 'info@panmind.com'
+
+    gemspec.add_dependency('eventmachine')
+    gemspec.add_dependency('couchrest')
+    gemspec.add_dependency('mongo')
+    gemspec.add_dependency('bson')
+    gemspec.add_dependency('bson_ext', '=1.6.0')
+  end
+rescue LoadError
+  puts 'Jeweler not available. Install it with: gem install jeweler'
+end
+
+desc 'Generate the rdoc'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_files.add %w( README.md lib/**/*.rb )
+
+  rdoc.main  = 'README.md'
+  rdoc.title = 'Usage Tracker Server'
+end

data/VERSION

@@ -0,0 +1 @@
+1.1.5

data/bin/usage_tracker_server

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'usage_tracker_server/runner'

data/config/usage_tracker_server.yml.sample

@@ -0,0 +1,18 @@
+# Available adapters: 'couchdb', 'mongodb'
+# adapter: 'couchdb'
+
+# CouchDB connect settings
+# database: "http://127.0.0.1:5984/usage_tracker"
+
+#  MongoDB connect settings
+adapter: 'mongodb'
+database:
+  name: "usage_tracker"
+  host: "127.0.0.1"
+  port: 27017
+  collection: "data"
+
+listen:  "127.0.0.1:5985"
+log_level: 'warn'
+
+

data/config/usage_tracker_server_upstart.conf

@@ -0,0 +1,14 @@
+description "Panmind Usage Tracker Server Daemon"
+author "Marcello Barnaba <marcello.barnaba@gmail.com>"
+version "1.1"
+
+start on runlevel [2345]
+stop on shutdown
+respawn
+
+# The following line assumes that you're using RVM, that's why
+# bash is invoked: to load rvm setup scripts.
+#
+# You should change the user and the directory
+#
+exec sudo -i -H -u panmind bash -c 'echo; cd panmind/deploy; exec usage_tracker_server'

data/lib/usage_tracker_server.rb

@@ -0,0 +1,94 @@
+require 'rubygems'
+require 'erb'
+require 'yaml'
+require 'pathname'
+require 'ostruct'
+require 'couchrest'
+require 'usage_tracker_server/log'
+require 'usage_tracker_server/adapter'
+
+module UsageTrackerServer
+  class << self
+    # Memoizes settings from the ./config/usage_tracker_server.yml file,
+    # relative from __FILE__ and searches for the "usage_tracker_server"
+    # configuration block. Raises RuntimeError if it cannot find
+    # the configuration.
+    #
+    def settings
+      @settings ||= begin
+
+        rc_file  = Pathname.new('.').join('config', 'usage_tracker_server.yml')
+        settings = YAML.load(rc_file.read) if rc_file.exist?
+
+        if settings == nil
+          raise "Configuration missing."
+        elsif settings.values_at(*%w(adapter database listen)).any?(&:nil?)
+          raise "Incomplete configuration: please set the 'adapter', 'database' and 'listen' keys"
+        end
+
+        host, port = settings.delete('listen').split(':')
+
+        if [host, port].any? {|x| x.strip.empty?}
+          raise "Please specify where to listen as host:port"
+        end
+
+        settings['host'], settings['port'] = host, port.to_i
+
+        settings['log_level'] ||= :warn
+        log.level = settings['log_level']
+
+        OpenStruct.new settings
+      end
+    end
+
+    def database
+      @database or raise "Not connected to the database"
+    end
+
+    def adapter
+      @adapter or raise "Not connected to the database adapter"
+    end
+
+    # Connects to the configured CouchDB and memoizes the
+    # CouchRest::Database connection into an instance variable
+    # and calls +load_views!+
+    #
+    # Raises RuntimeError if the connection could not be established
+    #
+    def connect!
+      @adapter = Adapter::new settings
+      @database = @adapter.database
+    end
+
+    # Code to run inside EventMachine
+    def run!
+      host, port = UsageTrackerServer.settings.host, UsageTrackerServer.settings.port
+
+      unless (1024..65535).include? port.to_i
+        raise "Please set a listening port between 1024 and 65535"
+      end
+
+      EventMachine.open_datagram_socket host, port, Reactor
+      log "Listening on #{host}:#{port} UDP"
+      write_pid!
+    end
+
+    def write_pid!(pid = $$)
+      dir = Pathname.new('.').join('tmp', 'pids')
+      dir = Pathname.new(Dir.tmpdir) unless dir.directory?
+      dir.join('usage_tracker_server.pid').open('w+') {|f| f.write(pid)}
+    end
+
+    def log(message = nil)
+      @log ||= Log.new
+      message ? @log.info(message) : @log
+    end
+
+    def raise(message)
+      log.error message
+      Kernel.raise Error, message
+    end
+  end
+
+  class Error < StandardError; end
+end

data/lib/usage_tracker_server/adapter.rb

@@ -0,0 +1,17 @@
+require 'usage_tracker_server/adapters/couchdb'
+require 'usage_tracker_server/adapters/mongodb'
+
+module UsageTrackerServer
+  class Adapter
+    def self::new(settings)
+      klass =
+        case settings.adapter
+          when 'couchdb'
+            Adapters::Couchdb
+          when 'mongodb'
+            Adapters::Mongodb
+        end
+      klass::new(settings)
+    end
+  end
+end

data/lib/usage_tracker_server/adapters/couchdb.rb

@@ -0,0 +1,35 @@
+require 'pathname'
+require 'couchrest'
+require 'yaml'
+
+module UsageTrackerServer
+  module Adapters
+    class Couchdb
+      attr_accessor :database
+      def initialize (settings)
+        @database =
+          CouchRest.database!(settings.database).tap do |db|
+            db.info
+          end
+      rescue Errno::ECONNREFUSED, RestClient::Exception => e
+        raise "Unable to connect to database #{settings.database}: #{e.message}"
+      end
+
+      def save_doc (doc)
+        doc['_id'] = make_id if doc['_id'].nil?
+        @database.save_doc(doc)
+      end
+
+      # Timestamp as _id has the advantage that documents
+      # are sorted automatically by CouchDB.
+      #
+      # Eventual duplication (multiple servers) is (possibly)
+      # avoided by adding a random digit at the end.
+      #
+      def make_id
+        Time.now.to_f.to_s.ljust(16, '0') + rand(10).to_s
+      end
+
+    end
+  end
+end

data/lib/usage_tracker_server/adapters/mongodb.rb

@@ -0,0 +1,28 @@
+require 'pathname'
+require 'mongo'
+require 'yaml'
+
+module UsageTrackerServer
+  module Adapters
+    class Mongodb
+      attr_accessor :database
+      def initialize (settings)
+        @database =
+          db = Mongo::Connection.new(settings.database['host'], settings.database['port']).db(settings.database['name'])
+
+          if settings.database['username'] || settings.database['password']
+            db.authenticate(settings.database['username'], settings.database['password'])
+          end
+
+          @collection = db[settings.database['collection']]
+          db
+      rescue Errno::ECONNREFUSED, Mongo::ConnectionError => e
+        raise "Unable to connect to database #{settings.database['name']} with #{settings.adapter} adapter: #{e.message}"
+      end
+
+      def save_doc(doc)
+        @collection.insert(doc)
+      end
+    end
+  end
+end

data/lib/usage_tracker_server/log.rb

@@ -0,0 +1,60 @@
+require 'logger'
+
+module UsageTrackerServer
+  class Log
+
+    Levels = {
+        :fatal => Logger::FATAL,
+        :error => Logger::ERROR,
+        :warn  => Logger::WARN,
+        :info  => Logger::INFO,
+        :debug => Logger::DEBUG
+    }
+
+    attr_reader :path
+
+    [:debug, :info, :warn, :error, :fatal].each do |severity|
+      define_method(severity) {|*args| @logger.send(severity, *args)}
+    end
+
+    def initialize
+      open
+    end
+
+    def path
+      @path ||= if File.directory?('log')
+        Pathname.new('.').join('log', 'usage_tracker_server.log')
+      else
+        Pathname.new('usage_tracker_server.log')
+      end
+    end
+
+    def open
+      @logger           = Logger.new(path.to_s)
+      @logger.formatter = Logger::Formatter.new
+      @logger.info 'Log opened'
+
+    rescue
+      raise Error, "Cannot open log file #{path}"
+    end
+
+    def close
+      return unless @logger
+
+      @logger.info 'Log closed'
+      @logger.close
+      @logger = nil
+    end
+
+    def rotate
+      close
+      open
+    end
+
+    def level=(level)
+      level = level.to_sym
+      raise "Invalid log level" unless Levels.keys.include?(level)
+      @logger.level = Levels[level]
+    end
+  end
+end

data/lib/usage_tracker_server/reactor.rb

@@ -0,0 +1,64 @@
+require 'usage_tracker_server'
+require 'json'
+
+module UsageTrackerServer
+  module Reactor
+    # This method is called upon every data reception
+    #
+    def receive_data(data)
+      doc = parse(data)
+      if doc && check(doc)
+        store(doc)
+      end
+    end
+
+    private
+      def parse(data)
+        JSON(data).tap {|h| h.reject! {|k,v| v.nil?}}
+      rescue JSON::ParserError
+        UsageTrackerServer.log.error "Tossing out invalid JSON #{data.inspect} (#{$!.message.inspect})"
+        return nil
+      end
+
+      def check(doc)
+        error =
+          if    !doc.kind_of?(Hash) then 'invalid'
+          elsif doc.empty?          then 'empty'
+          elsif !(missing = check_keys(doc)).empty?
+            "#{missing.join(', ')} missing"
+          end
+
+        if error
+          UsageTrackerServer.log.error "Tossing out invalid document #{doc.inspect}: #{error}"
+          return nil
+        else
+          return true
+        end
+      end
+
+      def check_keys(doc)
+        %w( duration env status ).reject {|k| doc.has_key?(k)}
+      end
+
+      def store(doc)
+        tries = 0
+
+        begin
+          UsageTrackerServer.log.debug "Received #{doc.inspect}"
+          UsageTrackerServer.adapter.save_doc(doc)
+
+        rescue RestClient::Conflict => e
+          if (tries += 1) < 10
+            UsageTrackerServer.log.warn "Retrying to save #{doc.inspect}, try #{tries}"
+            retry
+          else
+            UsageTrackerServer.log.error "Losing '#{doc.inspect}' because of too many conflicts"
+          end
+
+        rescue
+          UsageTrackerServer.log.error "Losing '#{doc.inspect}' because #$!"
+        end
+      end
+  end
+
+end

data/lib/usage_tracker_server/runner.rb

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby 
+
+# This is the runner script where EventMachine gets 
+# actually invocated.
+# Behaviour is defined in the UsageTrackerServer::Reactor module 
+
+$:<< File.expand_path(File.dirname(__FILE__) +  '/..')
+
+require 'usage_tracker_server'
+require 'usage_tracker_server/reactor'
+require 'eventmachine'
+
+module UsageTrackerServer
+
+  connect!
+
+  # Setup signal handlers
+  #
+  #  * INT, TERM: graceful exit
+  #  * USR1     : rotate logs
+  #
+  def self.sigexit(sig)
+    log "Received SIG#{sig}"
+    EventMachine.stop_event_loop
+  end
+
+  trap('INT')  { sigexit 'INT'  }
+  trap('TERM') { sigexit 'TERM' }
+  trap('USR1') { log.rotate  }
+
+  # Run the Event Loop
+  #
+  EventMachine.run do
+    begin
+
+      run! 
+
+      $stderr.puts "Started, logging to #{log.path}"
+      [$stdin, $stdout, $stderr].each {|io| io.reopen '/dev/null'}
+
+    rescue Exception => e
+      message = e.message == 'no datagram socket' ? "Unable to bind #{UsageTrackerServer.settings.host}:#{UsageTrackerServer.settings.port}" : e
+      log.fatal message
+      $stderr.puts message unless $stderr.closed?
+      EventMachine.stop_event_loop
+      exit 1
+    end
+  end
+
+  # Goodbye!
+  #
+  log 'Exiting'
+end 
+  

data/panmind-usage-tracker-server.gemspec

@@ -0,0 +1,66 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  require 'yaml'
+  YAML::ENGINE.yamler = 'syck'
+  s.name = "panmind-usage-tracker-server"
+  s.version = "1.1.5"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Christian Woerner", "Fabrizio Regini", "Marcello Barnaba"]
+  s.date = "2012-02-24"
+  s.description = "this software implements an EventMachine reactor which stores json-objects received via UDP in a database."
+  s.email = "info@panmind.com"
+  s.executables = ["usage_tracker_server"]
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/usage_tracker_server",
+    "config/usage_tracker_server.yml.sample",
+    "config/usage_tracker_server_upstart.conf",
+    "lib/usage_tracker_server.rb",
+    "lib/usage_tracker_server/adapter.rb",
+    "lib/usage_tracker_server/adapters/couchdb.rb",
+    "lib/usage_tracker_server/adapters/mongodb.rb",
+    "lib/usage_tracker_server/log.rb",
+    "lib/usage_tracker_server/reactor.rb",
+    "lib/usage_tracker_server/runner.rb",
+    "log/.placeholder",
+    "panmind-usage-tracker-server.gemspec"
+  ]
+  s.homepage = "http://github.com/Panmind/usage_tracker_server"
+  s.require_paths = ["lib"]
+  s.summary = "server component for usage-tracker-middleware (Rack)"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
+      s.add_runtime_dependency(%q<couchrest>, [">= 0"])
+      s.add_runtime_dependency(%q<mongo>, [">= 0"])
+      s.add_runtime_dependency(%q<bson>, [">= 0"])
+      s.add_runtime_dependency(%q<bson_ext>, ["= 1.6.0"])
+    else
+      s.add_dependency(%q<eventmachine>, [">= 0"])
+      s.add_dependency(%q<couchrest>, [">= 0"])
+      s.add_dependency(%q<mongo>, [">= 0"])
+      s.add_dependency(%q<bson>, [">= 0"])
+      s.add_dependency(%q<bson_ext>, ["= 1.6.0"])
+    end
+  else
+    s.add_dependency(%q<eventmachine>, [">= 0"])
+    s.add_dependency(%q<couchrest>, [">= 0"])
+    s.add_dependency(%q<mongo>, [">= 0"])
+    s.add_dependency(%q<bson>, [">= 0"])
+    s.add_dependency(%q<bson_ext>, ["= 1.6.0"])
+  end
+end
+

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification 
+name: panmind-usage-tracker-server
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.1.5
+platform: ruby
+authors: 
+- Christian Woerner
+- Fabrizio Regini
+- Marcello Barnaba
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-02-24 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: couchrest
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bson
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: bson_ext
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.6.0
+  type: :runtime
+  version_requirements: *id005
+description: this software implements an EventMachine reactor which stores json-objects received via UDP in a database.
+email: info@panmind.com
+executables: 
+- usage_tracker_server
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- README.md
+- Rakefile
+- VERSION
+- bin/usage_tracker_server
+- config/usage_tracker_server.yml.sample
+- config/usage_tracker_server_upstart.conf
+- lib/usage_tracker_server.rb
+- lib/usage_tracker_server/adapter.rb
+- lib/usage_tracker_server/adapters/couchdb.rb
+- lib/usage_tracker_server/adapters/mongodb.rb
+- lib/usage_tracker_server/log.rb
+- lib/usage_tracker_server/reactor.rb
+- lib/usage_tracker_server/runner.rb
+- log/.placeholder
+- panmind-usage-tracker-server.gemspec
+homepage: http://github.com/Panmind/usage_tracker_server
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: server component for usage-tracker-middleware (Rack)
+test_files: []
+