double_agent 0.1.2 → 0.2.0

data/CHANGELOG

@@ -1,3 +1,15 @@
+== Release 0.2.0 (July 30, 2011)
+
+* Seperate code into real modules
+
+* Only load parser and resources by default
+
+* Add subtotals to percentages_for
+
+* Add more data parsing methods to DoubleAgent::Logs::Entry objects
+
+* Various optimizations and stuff I forgot about
+
 == Release 0.1.2 (June 30, 2011)
 
 * Bugfix to #browser_sum and #os_sym returning nil on an empty or nil user agent. Should return :unkown.

data/README.rdoc

@@ -9,20 +9,17 @@ agent strings. It is designed for parsing large sets for review or analysis.
 
 == Loading
 
-Double Agent is broken up into four components - core, resources, stats, logs
+Double Agent is broken up into modules - Parser, Resources, Stats and Logs.
 
-    # Load the default components (core, resources and stats)
+    # Load the core modules (Parser, Resources)
     require 'double_agent'
 
-    # Load components individually (some have dependencies on others)
-    require 'double_agent/core|resources|stats|logs'
+    # Load modules individually
+    require 'double_agent/parser|resources|stats|logs|all'
 
-    # Load everything
-    require 'double_agent/all'
+= Parser
 
-= The Core Parser
-
-    ua_string = "pretent I'm a user agent string for Chrome on Ubuntu"
+    ua_string = "I'm a user agent string for Chrome 11 on Ubuntu"
 
     DoubleAgent.browser(ua_string)
     => "Chrome 11"
@@ -41,7 +38,7 @@ See the DoubleAgent module for more.
 = Resources
 
 DoubleAgent::Resource is a mix-in for objects with a user_agent method or attribute.
-It gives that class's objects all of the above methods.
+It gives that class's objects all of the above methods and more.
 
     class Login
       include DoubleAgent::Resource
@@ -61,29 +58,34 @@ It gives that class's objects all of the above methods.
 
 = Stats
 
-Calculate browser and OS shares for large sets of DoubleAgent::Resource objects with DoubleAgent::percentages_for.
+Calculate browser and OS shares for large sets of DoubleAgent::Resource objects with DoubleAgent::Stats::percentages_for.
 
     logins = Login.all
-    stats = DoubleAgent.percentages_for(logins, :browser_family, :os_family)
+    p logins.size
+    => 1000
 
+    stats = DoubleAgent::Stats.percentages_for(logins, :browser_family, :os_family)
     p stats
-    => [["Firefox", "Windows", 50.0], ["Internet Explorer", "Windows", 20.0], ["Safari", "OS X", 20.0], ["Firefox", "GNU/Linux", 10.0]]
+    => [["Firefox", "Windows", 50.0, 500], ["Internet Explorer", "Windows", 20.0, 200], ["Safari", "OS X", 20.0, 200], ["Firefox", "GNU/Linux", 10.0, 100]]
 
-    stats.each do |browser_family, os_family, percent|
-      puts "#{browser_family} on #{os_family} - #{percent}%"
+    stats.each do |browser_family, os_family, percent, real_num|
+      puts "#{browser_family} on #{os_family} - #{percent}% (#{real_num} hits)"
     end
+    => "Firefox on Windows - 50% (500 hits)"
+    => "Internet Explorer on Windows - 20% (200 hits)"
+    => "Safari on OS X - 20% (200 hits)"
+    => "Firefox on GNU/Linux - 10% (100 hits)"
 
 = Logs
 
-DoubleAgent::log_entries parses through Apache and Nginx access logs, instantiating each log line into a DoubleAgent::LogEntry
-object. It even reads gzipped logs (requires zlib)! Since the DoubleAgent::LogEntry class mixes in DoubleAgent::Resource, you
-can easily calculate browser and/or OS market share on your site.
-
-    require 'double_agent'
-    require 'double_agent/logs'
+DoubleAgent::Logs::entries parses through Apache and Nginx access logs, instantiating each log line into a DoubleAgent::Logs::Entry
+object. It even reads gzipped logs (requires zlib)! Since the DoubleAgent::Logs::Entry class mixes in DoubleAgent::Resource, you
+can easily calculate browser and/or OS share on your site.
 
-    entries = DoubleAgent.log_entries("/var/log/nginx/my-site.access.log*")
-    stats = DoubleAgent.percentages_for(entries, :browser)
+    entries = DoubleAgent::Logs.entries("/var/log/nginx/my-site.access.log*")
+    stats = DoubleAgent::Stats.percentages_for(entries, :browser)
 
     p stats
-    => [["Firefox 4", 20.0], ["Internet Explorer 8", 18.0], ...]
+    => [["Firefox 4", 20.0, 650], ["Internet Explorer 8", 18.0, 587], ...]
+
+Check out the DoubleAgent::Logs::Entry class for more methods besides user_agent.

data/lib/double_agent/all.rb

@@ -1,2 +1,4 @@
-require 'double_agent'
+require 'double_agent/parser'
+require 'double_agent/resources'
+require 'double_agent/stats'
 require 'double_agent/logs'

data/lib/double_agent/logs.rb

@@ -1,41 +1,102 @@
-require 'zlib'
+require 'date'
 
 module DoubleAgent
-  # Accepts a glob path like /var/logs/apache/my-site.access.log*,
-  # parses all matching files into an array of LegEntry objects, and returns them.
-  #
-  # options[:match] and options[:ignore] can each take a regular expression,
-  # ignoring lines that do and don't match, respectively.
-  def self.log_entries(glob_str, options={})
-    gz_regexp = /\.gz\Z/i
-    entries = []
-    parse = (options[:match] or options[:ignore]) \
-      ? lambda { |line| entries << LogEntry.new(line) if (options[:match].nil? or line =~ options[:match]) and (options[:ignore].nil? or line !~ options[:ignore]) } \
-      : lambda { |line| entries << LogEntry.new(line) }
-    Dir.glob(glob_str).each do |f|
-      File.open(f, 'r') do |file|
-        handle = f =~ gz_regexp ? Zlib::GzipReader.new(file) : file
-        handle.readlines.each &parse
+  # 
+  # The Logs module contains methods and classes for parsing user agent strings
+  # from Apache-style logs (includes default Nginx log format). Gzipped logs 
+  # are also supported.
+  # 
+  module Logs
+    begin
+      require 'zlib'
+      ZLIB = true
+    rescue LoadError
+      $stderr.puts "Zlib not available for DoubleAgent::Logs; gzipped log files will be skipped."
+      ZLIB = false
+    end
+
+    # This class represents a line in an Apache or Nginx access log.
+    # The user agent string is parsed out and available through the
+    # user_agent attribute, making it available to the mixed-in DoubleAgent::Resource methods.
+    # Datestamps and Timestamps may also be retrieved for each instance, using the #on and #at methods, respectively.
+
+    class Entry
+      include DoubleAgent::Resource 
+      # Returns the user agent string
+      attr_reader :user_agent, :line
+
+      # Regular expression for pulling a user agent string out of a log entry. It is rather imprecise
+      # only for efficiency's sake.
+      USER_AGENT_REGEXP = /" ".+$/
+
+      # Regexp for parsing an IP address
+      IP_REGEXP = /^[0-9a-z\.:]+/
+
+      # Regex for parsing the date out of the log line
+      DATESTAMP_REGEXP = %r{[0-9]+/[a-z]+/[0-9]+:}i
+      # Regex for parsing DATESTAMP_REGEXP into a Date object
+      DATESTAMP_FORMAT = '%d/%B/%Y:'
+
+      # Regex for parsing the datetime out of the log line
+      TIMESTAMP_REGEXP = %r{[0-9]+/[a-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+ (-|\+)[0-9]+}i
+      # Regex for parsing TIMESTAMP_REGEXP into a DateTime object
+      TIMESTAMP_FORMAT = '%d/%B/%Y:%H:%M:%S %z'
+
+      # Initializes a new Entry object. An Apache or Nginx log line should be
+      # passed to it.
+      def initialize(line)
+        @line = line
+        @user_agent = line.slice(USER_AGENT_REGEXP)
+      end
+
+      # Returns the IP address the hit originated from
+      def ip
+        @line.slice(IP_REGEXP)
+      end
+
+      # Returns the Date the hit occurred on
+      def on
+        date_str = @line.slice(DATESTAMP_REGEXP)
+        date_str ? Date.strptime(date_str, DATESTAMP_FORMAT) : nil
+      end
+
+      # Returns the DateTime the hit occurred at
+      def at
+        datetime_str = @line.slice(TIMESTAMP_REGEXP)
+        datetime_str ? DateTime.strptime(datetime_str, TIMESTAMP_FORMAT) : nil
       end
     end
-    entries
-  end
 
-  # This class represents a line in an Apache or Nginx access log.
-  # The user agent string is parsed out and available through the
-  # user_agent attribute, making it available to the mixed-in DoubleAgent::Resource.
-
-  class LogEntry
-    # Regular expression for pulling a user agent string out of a log entry
-    USER_AGENT_REGEXP = /" ".+$/
-    include DoubleAgent::Resource 
-    # Returns the user agent string
-    attr_reader :user_agent
-
-    # Initializes a new LogEntry object. An Apache or Nginx log line should be
-    # passed to it.
-    def initialize(line)
-      @user_agent = line.slice(USER_AGENT_REGEXP)
+    # Accepts a glob path like /var/logs/apache/my-site.access.log*,
+    # parses all matching files into an array of Entry objects, and returns them.
+    # Gzipped log files are parsed by Zlib.
+    #
+    # Options:
+    # 
+    # :match A regular expression. Only lines which match this will be returned.
+    # :ignore A regular expression. Any lines which match this will be ignored.
+    def self.entries(glob_str, options={})
+      match, ignore = options[:match], options[:ignore]
+      entries = []
+
+      # Define the parse lambda
+      parse = (match or ignore) \
+        ? lambda { |line| entries << Entry.new(line) unless (match and line !~ match) or (ignore and line =~ ignore) } \
+        : lambda { |line| entries << Entry.new(line) }
+
+      # Define the read lambda
+      read = lambda do |f|
+        zipped = f =~ /\.gz\Z/i
+        return unless ZLIB or not zipped
+        File.open(f, 'r') do |file|
+          handle = zipped ? Zlib::GzipReader.new(file) : file
+          #handle.each_line &parse # A little slower, but may be more memory efficient
+          handle.readlines.each &parse
+        end
+      end
+
+      Dir.glob(glob_str).each &read
+      entries
     end
   end
 end

data/lib/double_agent/{core.rb → parser.rb}

@@ -34,7 +34,7 @@ module DoubleAgent
       end
     end
 
-    # Returns the browser's name. If you provide an user agent string as an argument,
+    # Returns the browser's name. If you provide a user agent string as an argument,
     # it will attempt to also return the major version number. E.g. "Firefox 4".
     def browser(ua=nil)
       if ua and (@version or @safe_version)
@@ -48,7 +48,7 @@ module DoubleAgent
     # BrowserParser would return the Chromium BrowserParser. For browsers that are their 
     # own family (e.g. Firefox, IE) it will end up returning itself.
     def family
-      BROWSERS[family_sym]
+      BROWSERS[@family_sym]
     end
     
     private
@@ -56,7 +56,7 @@ module DoubleAgent
     # Attempts to parse and return the browser's version from a user agent string. Returns
     # nil if nothing is found.
     def version(ua)
-      if @safe_version and RUBY_VERSION < MIN_VERSION
+      if @safe_version
         ua.slice(@safe_version[0]).slice(@safe_version[1])
       else
         ua.slice(@version)
@@ -83,7 +83,7 @@ module DoubleAgent
     # OSParser would return the GNU/Linux OSerParser. For OSes that are their own
     # family (e.g. OS X) it will end up returning itself.
     def family
-      OSES[family_sym]
+      OSES[@family_sym]
     end
   end
 

data/lib/double_agent/stats.rb

@@ -1,49 +1,65 @@
 module DoubleAgent
-  # True if running under less than Ruby 1.9
-  BAD_RUBY = RUBY_VERSION < '1.9.0'
-
-  if BAD_RUBY
-    require 'bigdecimal'
-    # If BAD_RUBY, this is used in lieu of the native round method
-    def self.better_round(f, n)
-      d = BigDecimal.new f.to_s
-      d.round(n).to_f
-    end
-  end
-
-  # For the given "things", returns the share of the group that each attr has.
-  # 
-  # "things" is an array of objects who's classes "include DoubleAgent::Resource".
-  # 
-  # "args" is one or more method symbols from DoubleAgent::Resource.
   # 
-  # "args" may have, as it's last member, :threshold => n, where n is the lowest
-  # percentage you want returned.
+  # The Stats module provides methods for determining browser and/or OS share
+  # for large numbers of DoubleAgent::Resource objects.
   # 
-  # Example, Browser Family share:
-  # DoubleAgent.percentages_for(logins, :browser_family)
-  #   [['Firefox', 50.4], ['Chrome', 19.6], ['Internet Explorer', 15], ['Safari', 10], ['Unknown', 5]]
-  # 
-  # Example, Browser/OS share, asking for symbols back:
-  # DoubleAgent.percentages_for(server_log_entries, :browser_sym, :os_sym)
-  #   [[:firefox, :windows_7, 50.4], [:chrome, :osx, 19.6], [:msie, :windows_xp, 15], [:safari, :osx, 10], [:other, :other, 5]]
-  def self.percentages_for(things, *args)
-    options = args.last.is_a?(Hash) ? args.pop : {} # Break out options
-    p = {}
-    things.each do |h|
-      syms = args.map { |attr| h.send attr }
-      p[syms] ||= 0
-      p[syms] += 1
-    end
-    size = things.size.to_f
-    p = p.to_a
+  module Stats
+    # True if running under less than Ruby 1.9
+    BAD_RUBY = RUBY_VERSION < '1.9.0'
+
     if BAD_RUBY
-      p.collect! { |k,n| [*k.<<(better_round(((n * 100) / size), 2))] }
-    else
-      p.collect! { |k,n| [*k.<<(((n * 100) / size).round(2))] }
+      require 'bigdecimal'
+      # If BAD_RUBY, this is used in lieu of the native round method
+      def self.better_round(f, n)
+        d = BigDecimal.new f.to_s
+        d.round(n).to_f
+      end
+    end
+
+    # For the given "things", returns the share of the group that each attr has.
+    # 
+    # "things" is an array of objects who's classes mix-in DoubleAgent::Resource.
+    # 
+    # "args" is one or more method symbols from DoubleAgent::Resource.
+    # 
+    # "args" may have, as it's last member, :threshold => n, where n is the number of the lowest
+    # percentage you want returned.
+    # 
+    # Returns an array of [attribute(s), percent of total, number of total]
+    # 
+    # Example, Browser Family share:
+    # 
+    # DoubleAgent::Stats.percentages_for(logins, :browser_family)
+    #   [['Firefox', 50.4, 5040], ['Chrome', 19.6, 1960], ['Internet Explorer', 15, 1500], ['Safari', 10, 1000], ['Unknown', 5, 500]]
+    # 
+    # Example, Browser/OS share, asking for symbols back:
+    # 
+    # DoubleAgent::Stats.percentages_for(server_log_entries, :browser_sym, :os_sym)
+    #   [[:firefox, :windows_7, 50.4, 5040], [:chrome, :osx, 19.6, 1960], [:msie, :windows_xp, 15, 1500], [:safari, :osx, 10, 1000], [:other, :other, 5, 100]]
+    def self.percentages_for(things, *args)
+      options = args.last.is_a?(Hash) ? args.pop : {} # Break out options
+      results = {}
+      # Count each instance
+      things.each do |h|
+        syms = args.map { |attr| h.send attr }
+        results[syms] ||= 0
+        results[syms] += 1
+      end
+      size = things.size.to_f
+      results = results.to_a
+      # From the total, calculate the percentage held by each browser, OS, etc.
+      if BAD_RUBY
+        results.collect! { |k,n| [*k.<<(better_round(((n * 100) / size), 2)).<<(n)] }
+      else
+        # Ruby 1.9 syntax that blows up in Ruby 1.8
+        #results.collect! { |k,n| [*k, ((n * 100) / size).round(2), n] }
+        results.collect! { |k,n| [*k.<<(((n * 100) / size).round(2)).<<(n)] }
+      end
+      # Sort in ascending order
+      results.sort! { |a,b| b.last <=> a.last }
+      # Reject percentages below a specified threshold
+      results.reject! { |a| a[-2] < options[:threshold] } if options[:threshold]
+      results
     end
-    p.sort! { |a,b| b.last <=> a.last }
-    p.reject! { |a| a.last < options[:threshold] } if options[:threshold]
-    p
   end
 end

data/lib/double_agent.rb

@@ -1,3 +1,2 @@
-require 'double_agent/core'
+require 'double_agent/parser'
 require 'double_agent/resources'
-require 'double_agent/stats'

data/spec/data_spec.rb

@@ -0,0 +1,100 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+DA = DoubleAgent
+
+describe DoubleAgent do
+  # Internet Explorer
+  it 'should be Internet Explorer 10 on Windows 8' do
+    ua = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/5.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 10 on Windows 8'
+  end
+
+  it 'should be Internet Explorer 9 on Windows 7' do
+    ua = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 9 on Windows 7'
+  end
+
+  it 'should be Internet Explorer 8 on Windows Vista' do
+    ua = "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/5.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 8 on Windows Vista'
+  end
+
+  it 'should be Internet Explorer 7 on Windows XP' do
+    ua = "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.2; Win64; x64; Trident/5.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 7 on Windows XP'
+  end
+
+  it 'should be Internet Explorer 7 on Windows XP' do
+    ua = "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; Win64; x64; Trident/5.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 7 on Windows XP'
+  end
+
+  # Chrome
+  it 'should be Chrome 12 on Windows XP' do
+    ua = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.25 (KHTML, like Gecko) Chrome/12.0.706.0 Safari/534.25"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Chrome 12 on Windows XP'
+  end
+
+  # Chromium
+  it 'should be Chrome 12 on Ubuntu' do
+    ua = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.703.0 Chrome/12.0.703.0 Safari/534.24"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Chrome 12 on Ubuntu'
+  end
+
+  # Android
+  it 'should be Android 2.3 on Android' do
+    ua = "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Android 2.3 on Android'
+  end
+
+  # Safari
+  it 'should be Safari 5 on OS X' do
+    ua = "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Safari 5 on OS X'
+  end
+
+  # Opera
+  it 'should be Opera 11 on GNU/Linux' do
+    ua = "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Opera 11 on GNU/Linux'
+  end
+
+  it 'should be Opera 11 on Windows 7' do
+    ua = "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Opera 11 on Windows 7'
+  end
+
+  # Firefox
+  it 'should be Firefox 4 on GNU/Linux' do
+    ua = "Mozilla/5.0 (X11; U; Linux x86_64; pl-PL; rv:2.0) Gecko/20110307 Firefox/4.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Firefox 4 on GNU/Linux'
+  end
+
+  # Epiphany
+  it 'should be Epiphany on GNU/Linux' do
+    ua = "Mozilla/5.0 (X11; U; Linux x86_64; fr-FR) AppleWebKit/534.7 (KHTML, like Gecko) Epiphany/2.30.6 Safari/534.7"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Epiphany on GNU/Linux'
+  end
+
+  # Konqueror
+  it 'should be Konqueror on FreeBSD' do
+    ua = "Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on FreeBSD'
+  end
+
+  it 'should be Konqueror on Fedora' do
+    ua = "Mozilla/5.0 (compatible; Konqueror/4.4; Linux) KHTML/4.4.1 (like Gecko) Fedora/4.4.1-1.fc12"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on Fedora'
+  end
+
+  it 'should be Konqueror on Slackware' do
+    ua = "Mozilla/5.0 (compatible; Konqueror/4.2; Linux) KHTML/4.2.4 (like Gecko) Slackware/13.0"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on Slackware'
+  end
+
+  # BlackBerry
+  it 'should be BlackBerry on BlackBerry' do
+    ua = "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; zh-TW) AppleWebKit/534.8+ (KHTML, like Gecko) Version/6.0.0.448 Mobile Safari/534.8+"
+    "#{DA.browser ua} on #{DA.os ua}".should == 'BlackBerry on BlackBerry'
+  end
+end

data/spec/parser_spec.rb

@@ -1,100 +1,114 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-DA = DoubleAgent
-
 describe DoubleAgent do
-  # Internet Explorer
-  it 'should be Internet Explorer 10 on Windows 8' do
-    ua = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/5.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 10 on Windows 8'
-  end
-
-  it 'should be Internet Explorer 9 on Windows 7' do
-    ua = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 9 on Windows 7'
-  end
-
-  it 'should be Internet Explorer 8 on Windows Vista' do
-    ua = "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/5.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 8 on Windows Vista'
-  end
-
-  it 'should be Internet Explorer 7 on Windows XP' do
-    ua = "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.2; Win64; x64; Trident/5.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 7 on Windows XP'
-  end
-
-  it 'should be Internet Explorer 7 on Windows XP' do
-    ua = "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; Win64; x64; Trident/5.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Internet Explorer 7 on Windows XP'
-  end
-
-  # Chrome
-  it 'should be Chrome 12 on Windows XP' do
-    ua = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.25 (KHTML, like Gecko) Chrome/12.0.706.0 Safari/534.25"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Chrome 12 on Windows XP'
-  end
-
-  # Chromium
-  it 'should be Chrome 12 on Ubuntu' do
-    ua = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.703.0 Chrome/12.0.703.0 Safari/534.24"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Chrome 12 on Ubuntu'
-  end
-
-  # Android
-  it 'should be Android 2.3 on Android' do
-    ua = "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Android 2.3 on Android'
-  end
-
-  # Safari
-  it 'should be Safari 5 on OS X' do
-    ua = "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Safari 5 on OS X'
-  end
-
-  # Opera
-  it 'should be Opera 11 on GNU/Linux' do
-    ua = "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Opera 11 on GNU/Linux'
-  end
-
-  it 'should be Opera 11 on Windows 7' do
-    ua = "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Opera 11 on Windows 7'
-  end
-
-  # Firefox
-  it 'should be Firefox 4 on GNU/Linux' do
-    ua = "Mozilla/5.0 (X11; U; Linux x86_64; pl-PL; rv:2.0) Gecko/20110307 Firefox/4.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Firefox 4 on GNU/Linux'
-  end
-
-  # Epiphany
-  it 'should be Epiphany on GNU/Linux' do
-    ua = "Mozilla/5.0 (X11; U; Linux x86_64; fr-FR) AppleWebKit/534.7 (KHTML, like Gecko) Epiphany/2.30.6 Safari/534.7"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Epiphany on GNU/Linux'
-  end
-
-  # Konqueror
-  it 'should be Konqueror on FreeBSD' do
-    ua = "Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on FreeBSD'
-  end
-
-  it 'should be Konqueror on Fedora' do
-    ua = "Mozilla/5.0 (compatible; Konqueror/4.4; Linux) KHTML/4.4.1 (like Gecko) Fedora/4.4.1-1.fc12"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on Fedora'
-  end
-
-  it 'should be Konqueror on Slackware' do
-    ua = "Mozilla/5.0 (compatible; Konqueror/4.2; Linux) KHTML/4.2.4 (like Gecko) Slackware/13.0"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'Konqueror on Slackware'
-  end
-
-  # BlackBerry
-  it 'should be BlackBerry on BlackBerry' do
-    ua = "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; zh-TW) AppleWebKit/534.8+ (KHTML, like Gecko) Version/6.0.0.448 Mobile Safari/534.8+"
-    "#{DA.browser ua} on #{DA.os ua}".should == 'BlackBerry on BlackBerry'
+  context 'Parser' do
+    before do
+      @ua_string = 'Mozilla/5.0 (X11; Ubuntu Linux i686; rv:2.0) Gecko/20100101 Firefox/4.0'
+    end
+
+    #browser
+    it 'returns Firefox 4 for browser' do
+      DoubleAgent.browser(@ua_string).should == 'Firefox 4'
+    end
+    it 'returns Unknown for browser' do
+      DoubleAgent.browser('froofroo').should == 'Unknown'
+    end
+
+    #browser_sym
+    it 'returns :firefox for browser_sym' do
+      DoubleAgent.browser_sym(@ua_string).should == :firefox
+    end
+    it 'returns :unknown for browser_sym' do
+      DoubleAgent.browser_sym('froofroo').should == :unknown
+    end
+    it 'returns :unknown for an empty browser_sym' do
+      DoubleAgent.browser_sym('').should == :unknown
+    end
+    it 'returns :unknown for a nil browser_sym' do
+      DoubleAgent.browser_sym(nil).should == :unknown
+    end
+
+    #browser_family
+    it 'returns Firefox for browser family' do
+      DoubleAgent.browser_family(@ua_string).should == 'Firefox'
+    end
+
+    #browser_family_sym
+    it 'returns :firefox for browser_family_sym' do
+      DoubleAgent.browser_family_sym(@ua_string).should == :firefox
+    end
+    it 'returns :unknown for an empty browser_family_sym' do
+      DoubleAgent.browser_family_sym('').should == :unknown
+    end
+    it 'returns :unknown for a nil browser_family_sym' do
+      DoubleAgent.browser_family_sym(nil).should == :unknown
+    end
+
+    #browser_icon
+    it 'returns :firefox for browser_sym' do
+      DoubleAgent.browser_icon(@ua_string).should == :firefox
+    end
+    it 'returns :unkown for an empty browser_sym' do
+      DoubleAgent.browser_icon('').should == :unknown
+    end
+    it 'returns :unkown for a nil browser_sym' do
+      DoubleAgent.browser_icon(nil).should == :unknown
+    end
+
+    #browser_family_icon
+    it 'returns :firefox for browser_family_sym' do
+      DoubleAgent.browser_family_icon(@ua_string).should == :firefox
+    end
+    it 'returns :unkown for an empty browser_family_sym' do
+      DoubleAgent.browser_family_icon('').should == :unknown
+    end
+    it 'returns :unkown for a nil browser_family_sym' do
+      DoubleAgent.browser_family_icon(nil).should == :unknown
+    end
+
+    #os
+    it 'returns Ubuntua for OS' do
+      DoubleAgent.os(@ua_string).should == 'Ubuntu'
+    end
+    it 'returns Unknowna for OS' do
+      DoubleAgent.os('froofroo').should == 'Unknown'
+    end
+    it 'returns Unknowna for OS' do
+      DoubleAgent.os('').should == 'Unknown'
+    end
+
+    #os_sym
+    it 'returns :ubuntu for os_sym' do
+      DoubleAgent.os_sym(@ua_string).should == :ubuntu
+    end
+    it 'returns :unknown for os_sym' do
+      DoubleAgent.os_sym('froofroo').should == :unknown
+    end
+    it 'returns :unknown for an empty os_sym' do
+      DoubleAgent.os_sym('').should == :unknown
+    end
+    it 'returns :unknown for a nil os_sym' do
+      DoubleAgent.os_sym(nil).should == :unknown
+    end
+
+    #os_family
+    it 'returns GNU/Linux OS family' do
+      DoubleAgent.os_family(@ua_string).should == 'GNU/Linux'
+    end
+
+    #os_family_sym
+    it 'returns :linux for os_family_sym' do
+      DoubleAgent.os_family_sym(@ua_string).should == :linux
+    end
+
+    #os_icon
+    it 'returns :ubuntu for os_sym' do
+      DoubleAgent.os_icon(@ua_string).should == :ubuntu
+    end
+
+    #os_family_icon
+    it 'returns :linux for os_family_sym' do
+      DoubleAgent.os_family_icon(@ua_string).should == :linux
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
 require 'rspec'
-require File.dirname(__FILE__) + '/../lib/double_agent/core'
+require File.dirname(__FILE__) + '/../lib/double_agent/parser'
 require File.dirname(__FILE__) + '/../lib/double_agent/resources'
 require File.dirname(__FILE__) + '/../lib/double_agent/stats'
 require File.dirname(__FILE__) + '/../lib/double_agent/logs'
@@ -7,3 +7,13 @@ require File.dirname(__FILE__) + '/../lib/double_agent/logs'
 Rspec.configure do |c|
   c.mock_with :rspec
 end
+
+module Kernel
+  def suppress_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    result = yield
+    $VERBOSE = original_verbosity
+    return result
+  end
+end

data/spec/stats_spec.rb

@@ -1,7 +1,7 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
 log_glob = File.dirname(__FILE__) + '/data/*.access.log*'
-entries = DoubleAgent::log_entries(log_glob, :match => /^\d/)
+entries = DoubleAgent::Logs::entries(log_glob, :match => /^\d/)
 
 describe DoubleAgent do
   context 'Logs' do
@@ -10,27 +10,54 @@ describe DoubleAgent do
     end
 
     it 'should have loaded n log entries' do
-      DoubleAgent::log_entries(log_glob, :match => /^\d/, :ignore => %r{ /dashboard }).size.should == 44
+      DoubleAgent::Logs::entries(log_glob, :match => /^\d/, :ignore => %r{ /dashboard }).size.should == 44
+    end
+
+    context 'without zlib' do
+      it 'should have loaded n log entries' do
+        suppress_warnings { DoubleAgent::Logs::ZLIB = false }
+        plain_entries = DoubleAgent::Logs::entries(log_glob, :match => /^\d/)
+        suppress_warnings { DoubleAgent::Logs::ZLIB = true }
+        plain_entries.size.should == 17
+      end
+    end
+
+    context 'parsing other data' do
+      before :each do
+        @line = DoubleAgent::Logs::Entry.new '68.52.99.211 - - [04/May/2011:08:21:04 -0400] "GET / HTTP/1.1" 200 1312 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17"'
+      end
+
+      it 'should parse the IP' do
+        @line.ip.should == '68.52.99.211'
+      end
+
+      it 'should parse datestamps' do
+        @line.on.should == Date.new(2011, 5, 4)
+      end
+
+      it 'should parse timestamps' do
+        @line.at.should == DateTime.new(2011, 5, 4, 8, 21, 4, '-0400')
+      end
     end
   end
 
   context 'Stats' do
     it 'should calculate stats' do
-      stats = DoubleAgent.percentages_for entries, :browser_family, :os_family
-      answer = [["Internet Explorer", "Windows", 42.55],
-                ["Chromium", "GNU/Linux", 40.43],
-                ["Firefox", "GNU/Linux", 10.64],
-                ["Firefox", "OS X", 4.26],
-                ["Safari", "OS X", 2.13]]
+      stats = DoubleAgent::Stats.percentages_for entries, :browser_family, :os_family
+      answer = [["Internet Explorer", "Windows", 42.55, 20],
+                ["Chromium", "GNU/Linux", 40.43, 19],
+                ["Firefox", "GNU/Linux", 10.64, 5],
+                ["Firefox", "OS X", 4.26, 2],
+                ["Safari", "OS X", 2.13, 1]]
       stats.should == answer
     end
 
     it 'should ignore stats below the threshold' do
-      stats = DoubleAgent.percentages_for entries, :browser_family, :os_family, :threshold => 3.0
-      answer = [["Internet Explorer", "Windows", 42.55],
-                ["Chromium", "GNU/Linux", 40.43],
-                ["Firefox", "GNU/Linux", 10.64],
-                ["Firefox", "OS X", 4.26]]
+      stats = DoubleAgent::Stats.percentages_for entries, :browser_family, :os_family, :threshold => 3.0
+      answer = [["Internet Explorer", "Windows", 42.55, 20],
+                ["Chromium", "GNU/Linux", 40.43, 19],
+                ["Firefox", "GNU/Linux", 10.64, 5],
+                ["Firefox", "OS X", 4.26, 2]]
       stats.should == answer
     end
   end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
   - 2
-  version: 0.1.2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Jordan Hollinger
@@ -14,11 +14,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-30 00:00:00 -04:00
+date: 2011-07-30 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
-description: Browser User Agent string parser with resource, stats, and a log reader
+description: Browser User Agent string parser with resources, stats, and a log reader
 email: jordan@jordanhollinger.com
 executables: []
 
@@ -31,17 +31,17 @@ files:
 - lib/double_agent.rb
 - lib/double_agent/all.rb
 - lib/double_agent/logs.rb
-- lib/double_agent/core.rb
 - lib/double_agent/resources.rb
 - lib/double_agent/stats.rb
+- lib/double_agent/parser.rb
 - data/browsers.yml
 - data/oses.yml
 - spec/spec_helper.rb
-- spec/core_spec.rb
 - spec/data/httpd.access.log.1.gz
 - spec/data/httpd.access.log
 - spec/stats_spec.rb
 - spec/resources_spec.rb
+- spec/data_spec.rb
 - spec/parser_spec.rb
 - README.rdoc
 - LICENSE

data/spec/core_spec.rb

@@ -1,114 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe DoubleAgent do
-  context 'Core' do
-    before do
-      @ua_string = 'Mozilla/5.0 (X11; Ubuntu Linux i686; rv:2.0) Gecko/20100101 Firefox/4.0'
-    end
-
-    #browser
-    it 'returns Firefox 4 for browser' do
-      DoubleAgent.browser(@ua_string).should == 'Firefox 4'
-    end
-    it 'returns Unknown for browser' do
-      DoubleAgent.browser('froofroo').should == 'Unknown'
-    end
-
-    #browser_sym
-    it 'returns :firefox for browser_sym' do
-      DoubleAgent.browser_sym(@ua_string).should == :firefox
-    end
-    it 'returns :unknown for browser_sym' do
-      DoubleAgent.browser_sym('froofroo').should == :unknown
-    end
-    it 'returns :unknown for an empty browser_sym' do
-      DoubleAgent.browser_sym('').should == :unknown
-    end
-    it 'returns :unknown for a nil browser_sym' do
-      DoubleAgent.browser_sym(nil).should == :unknown
-    end
-
-    #browser_family
-    it 'returns Firefox for browser family' do
-      DoubleAgent.browser_family(@ua_string).should == 'Firefox'
-    end
-
-    #browser_family_sym
-    it 'returns :firefox for browser_family_sym' do
-      DoubleAgent.browser_family_sym(@ua_string).should == :firefox
-    end
-    it 'returns :unknown for an empty browser_family_sym' do
-      DoubleAgent.browser_family_sym('').should == :unknown
-    end
-    it 'returns :unknown for a nil browser_family_sym' do
-      DoubleAgent.browser_family_sym(nil).should == :unknown
-    end
-
-    #browser_icon
-    it 'returns :firefox for browser_sym' do
-      DoubleAgent.browser_icon(@ua_string).should == :firefox
-    end
-    it 'returns :unkown for an empty browser_sym' do
-      DoubleAgent.browser_icon('').should == :unknown
-    end
-    it 'returns :unkown for a nil browser_sym' do
-      DoubleAgent.browser_icon(nil).should == :unknown
-    end
-
-    #browser_family_icon
-    it 'returns :firefox for browser_family_sym' do
-      DoubleAgent.browser_family_icon(@ua_string).should == :firefox
-    end
-    it 'returns :unkown for an empty browser_family_sym' do
-      DoubleAgent.browser_family_icon('').should == :unknown
-    end
-    it 'returns :unkown for a nil browser_family_sym' do
-      DoubleAgent.browser_family_icon(nil).should == :unknown
-    end
-
-    #os
-    it 'returns Ubuntua for OS' do
-      DoubleAgent.os(@ua_string).should == 'Ubuntu'
-    end
-    it 'returns Unknowna for OS' do
-      DoubleAgent.os('froofroo').should == 'Unknown'
-    end
-    it 'returns Unknowna for OS' do
-      DoubleAgent.os('').should == 'Unknown'
-    end
-
-    #os_sym
-    it 'returns :ubuntu for os_sym' do
-      DoubleAgent.os_sym(@ua_string).should == :ubuntu
-    end
-    it 'returns :unknown for os_sym' do
-      DoubleAgent.os_sym('froofroo').should == :unknown
-    end
-    it 'returns :unknown for an empty os_sym' do
-      DoubleAgent.os_sym('').should == :unknown
-    end
-    it 'returns :unknown for a nil os_sym' do
-      DoubleAgent.os_sym(nil).should == :unknown
-    end
-
-    #os_family
-    it 'returns GNU/Linux OS family' do
-      DoubleAgent.os_family(@ua_string).should == 'GNU/Linux'
-    end
-
-    #os_family_sym
-    it 'returns :linux for os_family_sym' do
-      DoubleAgent.os_family_sym(@ua_string).should == :linux
-    end
-
-    #os_icon
-    it 'returns :ubuntu for os_sym' do
-      DoubleAgent.os_icon(@ua_string).should == :ubuntu
-    end
-
-    #os_family_icon
-    it 'returns :linux for os_family_sym' do
-      DoubleAgent.os_family_icon(@ua_string).should == :linux
-    end
-  end
-end