double_agent 0.0.3 → 0.1.1

data/CHANGELOG

@@ -1,3 +1,13 @@
+== Release 0.1.1 (May 10, 2011)
+
+* #log_entries takes :match and :ignore regexp options
+
+* Added :threshold option to #percentages_for
+
+* Massive performance improvements to log parsing
+
+* Minor performance improvements to user agent parsing
+
 == Release 0.0.3 (May 6, 2011)
 
 * Fixed bug from 0.0.2 in Ruby 1.8 where browser versions never got parsed

data/README.rdoc

@@ -3,35 +3,32 @@
 double_agent is a library for parsing browser and operating system info out of user 
 agent strings. It is designed for parsing large sets for review or analysis.
 
-    # Load core, resources and stats
+== Installation
+
+    gem install double_agent
+
+== Loading
+
+Double Agent is broken up into four components - core, resources, stats, logs
+
+    # Load the default components (core, resources and stats)
     require 'double_agent'
 
-    # Load pieces individually
-    require 'double_agent/core'
-    require 'double_agent/resources'
-    require 'double_agent/stats'
-    require 'double_agent/logs'
+    # Load components individually (some have dependencies on others)
+    require 'double_agent/core|resources|stats|logs'
 
-    # Load everything at once
+    # Load everything
     require 'double_agent/all'
 
-= Core
-
-The core parser.
+= The Core Parser
 
-    ua_string = "pretent I'm a user agent string for Firefox 4 on Ubuntu"
+    ua_string = "pretent I'm a user agent string for Chrome on Ubuntu"
 
     DoubleAgent.browser(ua_string)
-    => "Firefox 4"
+    => "Chrome 11"
 
     DoubleAgent.browser_family(ua_string)
-    => "Firefox"
-
-    DoubleAgent.browser_sym(ua_string)
-    => :firefox
-
-    DoubleAgent.browser_family_sym(ua_string)
-    => :firefox
+    => "Chromium"
 
     DoubleAgent.os(ua_string)
     => "Ubuntu"
@@ -39,22 +36,18 @@ The core parser.
     DoubleAgent.os_family(ua_string)
     => "GNU/Linux"
 
-    DoubleAgent.os_sym(ua_string)
-    => :ubuntu
-
-    DoubleAgent.os_family_sym(ua_string)
-    => :linux
+See the DoubleAgent module for more.
 
 = Resources
 
-Say you're saving a record of each user's login so you can determine their favorite browser. Just make sure
-the user agent is available through user_agent, and presto!
+DoubleAgent::Resource is a mix-in for objects with a user_agent method or attribute.
+It gives that class's objects all of the above methods.
 
     class Login
       include DoubleAgent::Resource
 
       def user_agent
-        #returns this object's user agent string
+        #returns the user agent string the user logged in with
       end
     end
 
@@ -68,21 +61,7 @@ the user agent is available through user_agent, and presto!
 
 = Stats
 
-Figure out what percent use which browser, browser family, os, etc.
-
-== Example 1
-
-    logins = Login.all
-    stats = DoubleAgent.percentages_for(logins, :browser)
-
-    p stats
-    => [["Firefox 4", 20.0], ["Internet Explorer 8", 18.0], ...]
-
-    stats.each do |browser, percent|
-      puts "#{browser} - #{percent}%"
-    end
-
-== Example 2
+Calculate browser and OS shares for large sets of DoubleAgent::Resource objects with DoubleAgent::percentages_for.
 
     logins = Login.all
     stats = DoubleAgent.percentages_for(logins, :browser_family, :os_family)
@@ -96,23 +75,15 @@ Figure out what percent use which browser, browser family, os, etc.
 
 = Logs
 
-DoubleAgent#log_entries parses through Apache and Nginx access logs, instantiating each log line into a LogEntry
-object. It even reads gzipped logs (requires zlib)! The LogEntry class mixes in DoubleAgent::Resource, so this is
-a great way to generate browser and OS breakdowns of who's visiting your site.
+DoubleAgent::log_entries parses through Apache and Nginx access logs, instantiating each log line into a DoubleAgent::LogEntry
+object. It even reads gzipped logs (requires zlib)! Since the DoubleAgent::LogEntry class mixes in DoubleAgent::Resource, you
+can easily calculate browser and/or OS market share on your site.
 
     require 'double_agent'
     require 'double_agent/logs'
 
     entries = DoubleAgent.log_entries("/var/log/nginx/my-site.access.log*")
-    entries.each do |entry|
-      puts entry.browser
-    end
+    stats = DoubleAgent.percentages_for(entries, :browser)
 
-    Firefox 3
-    Internet Explorer 8
-    Internet Explorer 9
-    Firefox 4
-    Internet Explorer
-    Safari
-    Chrome
-    ...
+    p stats
+    => [["Firefox 4", 20.0], ["Internet Explorer 8", 18.0], ...]

data/data/oses.yml

@@ -1,40 +1,3 @@
-- :name: Android
-  :sym: :android
-  :family_sym: :linux
-  :regex: android
-
-- :name: Ubuntu
-  :sym: :ubuntu
-  :family_sym: :linux
-  :regex: ubuntu
-
-- :name: Fedora
-  :sym: :fedora
-  :family_sym: :linux
-  :regex: fedora
-
-- :name: Slackware
-  :sym: :slackware
-  :family_sym: :linux
-  :regex: slackware
-
-- :name: GNU/Linux
-  :sym: :linux
-  :regex: linux
-
-- :name: FreeBSD
-  :sym: :freebsd
-  :regex: freebsd
-
-- :name: iOS
-  :sym: :ios
-  :family_sym: :osx
-  :regex: (iphone)|(ipad)
-
-- :name: OS X
-  :sym: :osx
-  :regex: macintosh
-
 - :name: Windows 8
   :sym: :windows_8
   :family_sym: :windows
@@ -63,6 +26,43 @@
   :sym: :windows
   :regex: windows
 
+- :name: iOS
+  :sym: :ios
+  :family_sym: :osx
+  :regex: (iphone)|(ipad)
+
+- :name: OS X
+  :sym: :osx
+  :regex: macintosh
+
+- :name: Android
+  :sym: :android
+  :family_sym: :linux
+  :regex: android
+
+- :name: Ubuntu
+  :sym: :ubuntu
+  :family_sym: :linux
+  :regex: ubuntu
+
+- :name: Fedora
+  :sym: :fedora
+  :family_sym: :linux
+  :regex: fedora
+
+- :name: Slackware
+  :sym: :slackware
+  :family_sym: :linux
+  :regex: slackware
+
+- :name: GNU/Linux
+  :sym: :linux
+  :regex: linux
+
+- :name: FreeBSD
+  :sym: :freebsd
+  :regex: freebsd
+
 - :name: BlackBerry
   :sym: :blackberry
   :regex: blackberry

data/lib/double_agent/logs.rb

@@ -4,16 +4,18 @@ module DoubleAgent
   # Accepts a glob path like /var/logs/apache/my-site.access.log*,
   # parses all matching files into an array of LegEntry objects, and returns them.
   #
-  # If a Regexp is passed as the second argument, lines which do not match
-  # it will be ignored.
-  def self.log_entries(glob_str, regex=nil)
-    entries, gz_regexp = [], /\.gz\Z/i
+  # options[:match] and options[:ignore] can each take a regular expression,
+  # ignoring lines that do and don't match, respectively.
+  def self.log_entries(glob_str, options={})
+    gz_regexp = /\.gz\Z/i
+    entries = []
+    parse = (options[:match] or options[:ignore]) \
+      ? lambda { |line| entries << LogEntry.new(line) if (options[:match].nil? or line =~ options[:match]) and (options[:ignore].nil? or line !~ options[:ignore]) } \
+      : lambda { |line| entries << LogEntry.new(line) }
     Dir.glob(glob_str).each do |f|
-      File.open(f) do |file|
+      File.open(f, 'r') do |file|
         handle = f =~ gz_regexp ? Zlib::GzipReader.new(file) : file
-        while ( line = handle.gets )
-          entries << LogEntry.new(line) if regex.nil? or line =~ regex
-        end
+        handle.readlines.each &parse
       end
     end
     entries
@@ -25,16 +27,14 @@ module DoubleAgent
 
   class LogEntry
     # Regular expression for pulling a user agent string out of a log entry
-    USER_AGENT_REGEXP = /[^"]+(?="$)/
-    include DoubleAgent::Resource
-
+    USER_AGENT_REGEXP = /" ".+$/
+    include DoubleAgent::Resource 
     # Returns the user agent string
     attr_reader :user_agent
 
     # Initializes a new LogEntry object. An Apache or Nginx log line should be
     # passed to it.
     def initialize(line)
-      #@line = line
       @user_agent = line.slice(USER_AGENT_REGEXP)
     end
   end

data/lib/double_agent/stats.rb

@@ -15,7 +15,10 @@ module DoubleAgent
   # 
   # "things" is an array of objects who's classes "include DoubleAgent::Resource".
   # 
-  # "attrs" is one or more method symbols from DoubleAgent::Resource.
+  # "args" is one or more method symbols from DoubleAgent::Resource.
+  # 
+  # "args" may have, as it's last member, :threshold => n, where n is the lowest
+  # percentage you want returned.
   # 
   # Example, Browser Family share:
   # DoubleAgent.percentages_for(logins, :browser_family)
@@ -24,11 +27,12 @@ module DoubleAgent
   # Example, Browser/OS share, asking for symbols back:
   # DoubleAgent.percentages_for(server_log_entries, :browser_sym, :os_sym)
   #   [[:firefox, :windows_7, 50.4], [:chrome, :osx, 19.6], [:msie, :windows_xp, 15], [:safari, :osx, 10], [:other, :other, 5]]
-  def self.percentages_for(things, *attrs)
+  def self.percentages_for(things, *args)
+    options = args.last.is_a?(Hash) ? args.pop : {} # Break out options
     p = {}
     things.each do |h|
-      syms = attrs.map { |attr| h.send attr }
-      p[syms] = 0 unless p.has_key? syms
+      syms = args.map { |attr| h.send attr }
+      p[syms] ||= 0
       p[syms] += 1
     end
     size = things.size.to_f
@@ -39,6 +43,7 @@ module DoubleAgent
       p.collect! { |k,n| [*k.<<(((n * 100) / size).round(2))] }
     end
     p.sort! { |a,b| b.last <=> a.last }
+    p.reject! { |a| a.last < options[:threshold] } if options[:threshold]
     p
   end
 end

data/spec/stats_spec.rb

@@ -1,22 +1,37 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
 log_glob = File.dirname(__FILE__) + '/data/*.access.log*'
-entries = DoubleAgent::log_entries(log_glob, /^\d/)
+entries = DoubleAgent::log_entries(log_glob, :match => /^\d/)
 
 describe DoubleAgent do
   context 'Logs' do
     it 'should have loaded n log entries' do
       entries.size.should == 47
     end
+
+    it 'should have loaded n log entries' do
+      DoubleAgent::log_entries(log_glob, :match => /^\d/, :ignore => %r{ /dashboard }).size.should == 44
+    end
   end
 
   context 'Stats' do
-    stats = DoubleAgent.percentages_for entries, :browser_family, :os_family
-    answer = [["Internet Explorer", "Windows", 42.55],
-              ["Chromium", "GNU/Linux", 40.43],
-              ["Firefox", "GNU/Linux", 10.64],
-              ["Firefox", "OS X", 4.26],
-              ["Safari", "OS X", 2.13]]
-    stats.should == answer
+    it 'should calculate stats' do
+      stats = DoubleAgent.percentages_for entries, :browser_family, :os_family
+      answer = [["Internet Explorer", "Windows", 42.55],
+                ["Chromium", "GNU/Linux", 40.43],
+                ["Firefox", "GNU/Linux", 10.64],
+                ["Firefox", "OS X", 4.26],
+                ["Safari", "OS X", 2.13]]
+      stats.should == answer
+    end
+
+    it 'should ignore stats below the threshold' do
+      stats = DoubleAgent.percentages_for entries, :browser_family, :os_family, :threshold => 3.0
+      answer = [["Internet Explorer", "Windows", 42.55],
+                ["Chromium", "GNU/Linux", 40.43],
+                ["Firefox", "GNU/Linux", 10.64],
+                ["Firefox", "OS X", 4.26]]
+      stats.should == answer
+    end
   end
 end

metadata

@@ -2,7 +2,7 @@
 name: double_agent
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.3
+  version: 0.1.1
 platform: ruby
 authors: 
 - Jordan Hollinger
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-30 00:00:00 Z
+date: 2011-05-10 00:00:00 Z
 dependencies: []
 
 description: Browser User Agent string parser with resource, stats, and a log reader