mongoid_traffic 0.0.1

data/lib/mongoid_traffic/controller_additions.rb

@@ -0,0 +1,32 @@
+module MongoidTraffic
+  module ControllerAdditions
+
+    def log_traffic scope: nil
+      MongoidTraffic::Logger.log(
+        ip_address: request.remote_ip,
+        referer: request.headers['Referer'],
+        unique_id: request.session_options[:id], # FIXME: not sure about this
+        user_agent: request.headers['User-Agent']
+      )
+    end
+    
+    def log_scoped_traffic scope: nil
+      log_traffic(scope: (scope || request.fullpath.split('?').first))
+    end
+
+    # ---------------------------------------------------------------------
+    
+    def self.included base
+      base.extend ClassMethods
+      base.helper_method :log_traffic
+      base.helper_method :log_scoped_traffic
+    end
+
+  end
+end
+
+if defined? ActionController::Base
+  ActionController::Base.class_eval do
+    include MongoidTraffic::ControllerAdditions
+  end
+end

data/lib/mongoid_traffic/log.rb

@@ -0,0 +1,86 @@
+module MongoidTraffic
+  class Log
+
+    include Mongoid::Document
+
+    # ---------------------------------------------------------------------
+    
+    field :s, as: :scope, type: String
+
+    field :ac, as: :access_count, type: Integer
+
+    field :b, as: :browsers, type: Hash, default: {}
+    field :c, as: :countries, type: Hash, default: {}
+    field :r, as: :referers, type: Hash, default: {}
+    field :u, as: :unique_ids, type: Hash, default: {}
+
+    field :uat, as: :updated_at, type: Time
+
+    # ---------------------------------------------------------------------
+    
+    field :df, as: :date_from, type: Date
+    field :dt, as: :date_to, type: Date
+
+    # ---------------------------------------------------------------------
+    
+    validates :date_from, presence: true
+    validates :date_to, presence: true
+
+    # ---------------------------------------------------------------------
+    
+    default_scope -> { where(scope: nil) }
+
+    scope :for_dates, -> date_from, date_to { where(date_from: date_from, date_to: date_to) }
+
+    scope :yearly, -> year { self.for_dates(Date.parse("01/01/#{year}"), Date.parse("01/01/#{year}").at_end_of_year) }
+    scope :monthly, -> month, year { self.for_dates(Date.parse("01/#{month}/#{year}"), Date.parse("01/#{month}/#{year}").at_end_of_month) }
+    scope :weekly, -> week, year { self.for_dates(Date.commercial(year, week), Date.commercial(year, week).at_end_of_week) }
+    scope :daily, -> date { self.for_dates(date, date) }
+
+    scope :scoped_to, -> scope { where(scope: scope) }
+    
+    # ---------------------------------------------------------------------
+    
+    index({ scope: 1, date_from: 1, date_to: 1 })
+
+    # =====================================================================
+    
+    def self.aggregate_on att
+      case find_field_by_name(att).type.to_s
+      when 'Integer' then sum(att)
+      when 'Hash' then sum_hash(att)
+      end
+    end
+
+    def self.sum att
+      if att.to_sym == :unique_ids
+        aggregate_on(:unique_ids).keys.count
+      else
+        super(att)
+      end
+    end
+
+    private # =============================================================
+    
+    def self.find_field_by_name field_name
+      return unless f = fields.detect{ |k,v| k == field_name.to_s or v.options[:as].to_s == field_name.to_s }
+      f.last
+    end
+
+    def self.sum_hash field_name
+      self.pluck(field_name).inject({}) do |res, h|
+        merger = proc { |key, v1, v2| 
+          if Hash === v1 && Hash === v2
+            v1.merge(v2, &merger)
+          elsif Hash === v2
+            v2
+          else
+            v1.to_i + v2.to_i
+          end
+        }
+        res = res.merge(h, &merger)
+      end
+    end
+
+  end
+end

data/lib/mongoid_traffic/logger/bots.rb

@@ -0,0 +1,34 @@
+require 'nokogiri'
+
+# sourced from https://github.com/charlotte-ruby/impressionist/blob/master/lib/impressionist/bots.rb
+
+module MongoidTraffic
+  class Logger
+    class Bots
+
+      DATA_URL = "http://www.user-agents.org/allagents.xml"
+      FILE_PATH = "vendor/mongoid_traffic/allagents.xml"
+      
+      class << self
+        def list
+          @list ||= begin
+            response = File.open(FILE_PATH).read
+            doc = Nokogiri::XML(response)
+            list = []
+            doc.xpath('//user-agent').each do |agent|
+              type = agent.xpath("Type").text
+              list << agent.xpath('String').text.gsub("&lt;","<") if %w(R S).include?(type)
+            end
+            list
+          end
+        end
+
+        def is_a_bot? referer
+          return false unless referer.present?
+          list.include?(referer)
+        end
+      end
+
+    end
+  end
+end

data/lib/mongoid_traffic/logger/browser.rb

@@ -0,0 +1,31 @@
+module MongoidTraffic
+  class Logger
+    class Browser
+
+      def initialize user_agent_string
+        @user_agent_string = user_agent_string
+      end
+
+      # ---------------------------------------------------------------------
+      
+      def platform
+        user_agent.platform
+      end
+
+      def name
+        user_agent.browser
+      end
+
+      def version
+        user_agent.version.to_s.split('.')[0..1].join('.')
+      end
+
+      private # =============================================================
+
+      def user_agent
+        @user_agent ||= ::UserAgent.parse(@user_agent_string)
+      end
+
+    end
+  end
+end

data/lib/mongoid_traffic/logger/geo_ip.rb

@@ -0,0 +1,22 @@
+require 'geoip'
+
+module MongoidTraffic
+  class Logger
+    class GeoIp
+
+      DATA_URL = "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
+      FILE_URL = "vendor/mongoid_traffic/GeoIP.dat"
+
+      class << self
+        def geoip
+          @geoip ||= ::GeoIP.new(FILE_URL)
+        end
+
+        def country_code2 str
+          geoip.country(str).country_code2
+        end
+      end
+
+    end
+  end
+end

data/lib/mongoid_traffic/logger/referer.rb

@@ -0,0 +1,27 @@
+require 'uri'
+
+module MongoidTraffic
+  class Logger
+    class Referer
+
+      def initialize referer_string
+        @referer_string = referer_string
+      end
+
+      def host
+        uri.host
+      end
+
+      def to_s
+        @referer_string
+      end
+
+      private # =============================================================
+      
+      def uri
+        URI.parse(@referer_string)
+      end
+
+    end
+  end
+end

data/lib/mongoid_traffic/logger.rb

@@ -0,0 +1,125 @@
+require 'geoip'
+require 'uri'
+require 'useragent'
+
+require_relative './log'
+require_relative './logger/bots'
+require_relative './logger/browser'
+require_relative './logger/geo_ip'
+require_relative './logger/referer'
+
+module MongoidTraffic
+  class Logger
+
+    TIME_SCOPE_OPTIONS = %i(year month week day)
+
+    # ---------------------------------------------------------------------
+    
+    def self.log *args
+      new(*args).log
+    end
+
+    # ---------------------------------------------------------------------
+
+    def initialize ip_address: nil, referer: nil, scope: nil, time_scope: %i(month day), unique_id: nil, user_agent: nil 
+      @ip_address = ip_address
+      @referer_string = referer
+      @scope = scope
+      @time_scope = time_scope
+      @unique_id = unique_id
+      @user_agent_string = user_agent
+    end
+
+    def log
+      return if Bots.is_a_bot?(@referer_string)
+      raise "Invalid time scope definition: #{@time_scope}" unless @time_scope.all?{ |ts| TIME_SCOPE_OPTIONS.include?(ts) }
+
+      @time_scope.each do |ts|
+        Log.collection.find( find_query(ts) ).upsert( upsert_query )
+      end
+    end
+
+    # ---------------------------------------------------------------------
+
+    def upsert_query
+      { 
+        '$inc' => access_count_query.
+          merge(browser_query).
+          merge(country_query).
+          merge(referer_query).
+          merge(unique_id_query),
+        '$set' => { uat: Time.now }
+      }
+    end
+
+    # ---------------------------------------------------------------------
+
+    def access_count_query
+      { ac: 1 }
+    end
+
+    def browser_query
+      return {} unless browser.present?
+      browser_path = [browser.platform, browser.name, browser.version].map{ |s| escape_key(s) }.join('.')
+      { "b.#{browser_path}" => 1 }
+    end
+
+    def country_query
+      return {} unless @ip_address.present?
+      return {} unless country_code2 = GeoIp.country_code2(@ip_address)
+      country_code_key = escape_key(country_code2)
+      { "c.#{country_code_key}" => 1 }
+    end
+
+    def referer_query
+      return {} unless referer.present?
+      referer_key = escape_key(referer.to_s)
+      { "r.#{referer_key}" => 1 }
+    end
+
+    def unique_id_query
+      return {} unless @unique_id.present?
+      unique_id_key = escape_key(@unique_id.to_s)
+      { "u.#{unique_id_key}" => 1 }
+    end
+
+    # ---------------------------------------------------------------------
+    
+    def escape_key key
+      CGI::escape(key).gsub('.', '%2E')
+    end
+
+    # ---------------------------------------------------------------------
+
+    def find_query ts
+      res = time_query(ts)
+      res = res.merge(scope_query) if @scope.present?
+      res
+    end
+
+    def scope_query
+      { s: @scope }
+    end
+
+    def time_query ts
+      date = Date.today
+      case ts
+      when :day then { df: date, dt: date }
+      else { df: date.send("at_beginning_of_#{ts}"), dt: date.send("at_end_of_#{ts}") }
+      end
+    end
+
+    private # =============================================================
+    
+    def browser
+      return unless @user_agent_string.present?
+      @browser ||= Browser.new(@user_agent_string)
+    end
+
+    def referer
+      return unless @referer_string.present?
+      @referer ||= Referer.new(@referer_string)
+    end
+
+  end
+end

data/lib/mongoid_traffic/version.rb

@@ -0,0 +1,3 @@
+module MongoidTraffic
+  VERSION = "0.0.1"
+end

data/lib/mongoid_traffic.rb

@@ -0,0 +1,3 @@
+require "mongoid_traffic/version"
+
+require "mongoid_traffic/controller_additions"

data/mongoid_traffic.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mongoid_traffic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mongoid_traffic"
+  spec.version       = MongoidTraffic::VERSION
+  spec.authors       = ["Tomas Celizna"]
+  spec.email         = ["tomas.celizna@gmail.com"]
+  spec.description   = %q{Aggregated traffic logs stored in MongoDB.}
+  spec.summary       = %q{Aggregated traffic logs stored in MongoDB.}
+  spec.homepage      = "https://github.com/tomasc/mongoid_traffic"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "geoip"
+  spec.add_dependency "mongoid", "~> 4.0"
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "useragent", "~> 0.10.0"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-minitest"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rake"
+end

data/test/mongoid_traffic/controller_additions_test.rb

@@ -0,0 +1,27 @@
+require 'test_helper'
+
+require_relative '../../lib/mongoid_traffic/controller_additions'
+
+module MongoidTraffic
+  describe 'ControllerAdditions' do
+
+    before(:each) do
+      @controller_class = Class.new
+      @controller = @controller_class.new
+      def @controller_class.helper_method(name); end
+      @controller_class.send(:include, MongoidTraffic::ControllerAdditions)
+    end
+
+    describe '.log_traffic' do
+      it 'logs with :user_agent'
+      it 'logs with :referer'
+      it 'logs with :ip_address'
+      it 'logs with :unique_id'
+    end
+
+    describe '.log_scoped_traffic' do
+      it 'infers scope from request path'
+    end
+
+  end
+end

data/test/mongoid_traffic/log_test.rb

@@ -0,0 +1,153 @@
+require 'test_helper'
+
+require_relative '../../lib/mongoid_traffic/log'
+
+module MongoidTraffic
+  describe 'Log' do
+    subject { Log.new }
+
+    describe 'fields' do
+      it 'has :scope' do
+        subject.must_respond_to :scope
+      end
+      it 'has :access_count' do
+        subject.must_respond_to :access_count
+      end
+      it 'has :browsers' do
+        subject.must_respond_to :browsers
+        subject.browsers.must_be_kind_of Hash
+      end
+      it 'has :referers' do
+        subject.must_respond_to :referers
+        subject.referers.must_be_kind_of Hash
+      end
+      it 'has :countries' do
+        subject.must_respond_to :countries
+        subject.countries.must_be_kind_of Hash
+      end
+      it 'has :unique_ids' do
+        subject.must_respond_to :unique_ids
+        subject.unique_ids.must_be_kind_of Hash
+      end
+      it 'has :updated_at' do
+        subject.must_respond_to :updated_at
+      end
+    end
+
+    describe 'scopes' do
+      it 'has :default_scope that assumes no :scope' do
+        Log.criteria.selector.fetch('s').must_be_nil
+      end
+
+      it('has :for_dates') { Log.must_respond_to :for_dates }
+      it('has :yearly') { Log.must_respond_to :yearly }
+      it('has :monthly') { Log.must_respond_to :monthly }
+      it('has :weekly') { Log.must_respond_to :weekly }
+      it('has :daily') { Log.must_respond_to :daily }
+      it('has :scoped_to') { Log.must_respond_to :scoped_to }
+    end
+
+    describe '.aggregate_on' do
+      let(:log_1) { Log.new(date_from: Date.today, date_to: Date.today) }
+      let(:log_2) { Log.new(date_from: Date.tomorrow, date_to: Date.tomorrow) }
+      
+      describe '.aggregate_on(:access_count)' do
+        before do
+          log_1.tap{ |l| l.access_count = 1 }.save
+          log_2.tap{ |l| l.access_count = 2 }.save
+        end
+
+        it 'sums the access_counts' do
+          Log.aggregate_on(:access_count).must_equal 3
+        end
+      end
+
+      describe '.aggregate_on(:browsers)' do
+        # not the key names have been abbreviated
+        before do
+          log_1.tap do |l| 
+            l.browsers = {
+              "Mac" => {
+                "Saf" => { "8" => 1 } 
+              }, 
+              "Win" => { 
+                "Saf" => { "7" => 5 } 
+              }
+            }
+          end.save
+          
+          log_2.tap do |l| 
+            l.browsers = { 
+              "Mac" => { 
+                "Saf" => { "8" => 10, "7" => 100 }, 
+                "Chr" => { "3" => 5 } 
+              }, 
+              "Win" => { 
+                "Saf" => { "7" => 100 }, 
+                "IE" => { "10" => 1 } 
+              } 
+            }
+          end.save
+        end
+
+        it 'sums the browsers' do
+          Log.aggregate_on(:browsers).must_equal({
+            "Mac" => { 
+              "Saf" => { "8" => 11, "7" => 100 }, 
+              "Chr" => { "3" => 5 } 
+            }, 
+            "Win" => { 
+              "Saf" => { "7" => 105 }, 
+              "IE" => { "10" => 1 } 
+            }
+          })
+        end
+      end
+
+      describe '.aggregate_on(:referers)' do
+        before do
+          log_1.tap{ |l| l.referers = { 'google' => 100, 'apple' => 1000 } }.save
+          log_2.tap{ |l| l.referers = { 'google' => 10, 'apple' => 100, 'ms' => 1 } }.save
+        end
+
+        it 'sums the referers' do
+          Log.aggregate_on(:referers).must_equal({ 'google' => 110, 'apple' => 1100, 'ms' => 1 })
+        end
+      end
+
+      describe '.aggregate_on(:countries)' do
+        before do
+          log_1.tap{ |l| l.countries = { 'CZ' => 100 } }.save
+          log_2.tap{ |l| l.countries = { 'DE' => 10 } }.save
+        end
+
+        it 'sums the countries' do
+          Log.aggregate_on(:countries).must_equal({ 'CZ' => 100, 'DE' => 10 })
+        end
+      end
+
+      describe '.aggregate_on(:unique_ids)' do
+        before do
+          log_1.tap{ |l| l.unique_ids = { '01234' => 100, '56789' => 100 } }.save
+          log_2.tap{ |l| l.unique_ids = { '56789' => 100 } }.save
+        end
+
+        it 'sums the unique_ids' do
+          Log.aggregate_on(:unique_ids).must_equal({ '01234' => 100, '56789' => 200 })
+        end
+      end
+
+      describe '.sum(:unique_ids)' do
+        before do
+          log_1.tap{ |l| l.unique_ids = { '01234' => 100, '56789' => 100 } }.save
+          log_2.tap{ |l| l.unique_ids = { '56789' => 100, 'ABCDE' => 1 } }.save
+        end
+
+        it 'sums the unique_ids' do
+          Log.sum(:unique_ids).must_equal 3
+        end
+      end
+    end
+
+  end
+end