useragent_parser 0.1.1 → 0.2.0

data/config/regexes.yaml

@@ -79,8 +79,31 @@ user_agent_parsers:
 
   # Lightning (for Thunderbird)
   # http://www.mozilla.org/projects/calendar/lightning/
+  - regex: 'Lightning/\d+\.\d+[ab]?\d+[a-z]* (Thunderbird)/(\d+)\.(\d+)\.?(\d+)?'
+
   - regex: '(Lightning)/(\d+)\.(\d+)([ab]?\d+[a-z]*)'
 
+  # Eudora
+  - regex: '(Eudora)/(\d+)\.(\d+)\.?(\d+)?'
+
+  - regex: '(EUDORA)'
+    family_replacement: 'Eudora'
+
+  # T-Online E-Mail
+  - regex: '(T-Online eMail) (\d+)\.(\d+)?'
+
+  # Apple Mail
+  - regex: '(Mail)/\d+.*Darwin/10\.5'
+    family_replacement: 'Apple Mail'
+    v1_replacement: '3'
+
+  # Sparrow
+  - regex: '(Sparrow)/\d+'
+    family_replacement: 'Sparrow'
+
+  # Lotus Notes
+  - regex: '(Lotus-Notes)/(\d+)\.(\d+)'
+    family_replacement: 'Lotus Notes'
   # Swiftfox
   - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+(?:pre)?) \(Swiftfox\)'
     family_replacement: 'Swiftfox'
@@ -153,21 +176,29 @@ user_agent_parsers:
   - regex: '(Twitterbot)/(\d+)\.(\d+)'
     family_replacement: 'TwitterBot'
 
+  # Windows Live Mail / Outlook Express
+  - regex: '(Outlook-Express)/(\d+)\.(\d+)'
+    family_replacement: 'Windows Live Mail'
+
   #### END SPECIAL CASES TOP ####
 
   #### MAIN CASES - this catches > 50% of all browsers ####
 
   # Browser/major_version.minor_version.beta_version
-  - regex: '(AdobeAIR|Chromium|FireWeb|Jasmine|ANTGalio|Midori|Fresco|Lobo|PaleMoon|Maxthon|Lynx|OmniWeb|Dillo|Camino|Demeter|Fluid|Fennec|Shiira|Sunrise|Chrome|Flock|Netscape|Lunascape|WebPilot|Vodafone|NetFront|Netfront|Konqueror|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|Opera Mini|iCab|NetNewsWire|ThunderBrowse|Iris|UP\.Browser|Bunjaloo|Google Earth|Raven for Mac)/(\d+)\.(\d+)\.(\d+)'
+  - regex: '(AdobeAIR|Chromium|FireWeb|Jasmine|ANTGalio|Midori|Fresco|Lobo|PaleMoon|Maxthon|Lynx|OmniWeb|Dillo|Camino|Demeter|Fluid|Fennec|Shiira|Sunrise|Chrome|Flock|Netscape|Lunascape|WebPilot|Vodafone|NetFront|Netfront|Konqueror|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|Opera Mini|iCab|NetNewsWire|Thunderbird|ThunderBrowse|Iris|UP\.Browser|Bunjaloo|Google Earth|Raven for Mac)/(\d+)\.(\d+)\.(\d+)'
 
   # Browser/major_version.minor_version
-  - regex: '(Bolt|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Chrome|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|Vodafone|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser)/(\d+)\.(\d+)'
+  - regex: '(Bolt|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Chrome|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|Vodafone|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Thunderbird)/(\d+)\.(\d+)'
 
   # Browser major_version.minor_version.beta_version (space instead of slash)
   - regex: '(iRider|Crazy Browser|SkipStone|iCab|Lunascape|Sleipnir|Maemo Browser) (\d+)\.(\d+)\.(\d+)'
   # Browser major_version.minor_version (space instead of slash)
   - regex: '(iCab|Lunascape|Opera|Android|Jasmine|Polaris|BREW) (\d+)\.(\d+)\.?(\d+)?'
 
+  # Microsoft Outlook 2007/2010
+  - regex: '(MSOffice) (\d+)'
+    family_replacement: 'Outlook'
+
   # weird android UAs
   - regex: '(Android) Donut'
     v1_replacement: '1'
@@ -319,6 +350,10 @@ user_agent_parsers:
 
   - regex: '(Nintendo 3DS).* Version/(\d+)\.(\d+)(?:\.(\w+))'
 
+  # Generic AppleWebKit detection
+  - regex: '(AppleWebKit)/(\d+)\.?(\d+)?.*\(KHTML, like Gecko\)'
+    family_replacement: 'Apple WebKit'
+
 os_parsers:
 
   ##########
@@ -827,3 +862,38 @@ mobile_os_families:
   - 'Windows Phone 6.5'
   - 'Windows CE'
   - 'Symbian OS'
+
+referrer_parsers:
+  - regex: '(mail\.live\.com)'
+    referrer_replacement: 'Hotmail'
+
+  - regex: '(mail\.yahoo\.com)'
+    referrer_replacement: 'Yahoo! Mail'
+
+  - regex: '(email\.freenet\.de)'
+    referrer_replacement: 'WEB.DE'
+
+  - regex: '(mail\.aol\.com)'
+    referrer_replacement: 'AOL Webmail'
+
+  - regex: '(mail\.google\.com)'
+    referrer_replacement: 'Gmail'
+
+  - regex: '(win\.mail\.ru)'
+    referrer_replacement: 'mail.ru'
+
+  - regex: '(service\.gmx\.net)'
+    referrer_replacement: 'GMX'
+
+  - regex: '(www\.gmxattachments\.net)'
+    referrer_replacement: 'GMX'
+
+  - regex: '(proxy[^\.]+\.bluewin\.ch)'
+    referrer_replacement: 'Swisscom'
+
+  - regex: '(communicator\.strato\.de)'
+    referrer_replacement: 'Strato'
+
+  - regex: '.+'
+    referrer_replacement: 'Other'
+

data/lib/useragent_parser/parsers/referrer_parser.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+module UseragentParser
+  class ReferrerParser
+    attr_accessor :pattern, :referrer_re, :referrer_replacement
+
+    def initialize(pattern, referrer_replacement = nil)
+      @pattern = pattern
+      @referrer_re = Regexp.compile(pattern)
+      @referrer_replacement = referrer_replacement
+    end
+
+    def match_spans(referrer_string)
+      match_spans = []
+      match = @referrer_re.match(referrer_string)
+      if match
+        # Return the offsets
+      end
+    end
+
+    def parse(referrer_string)
+      referrer = nil
+      match = @referrer_re.match(referrer_string)
+      if match
+        if @referrer_replacement
+          referrer = @referrer_replacement
+        else
+          referrer = match[1]
+        end
+      end
+      return referrer
+    end
+  end
+end

data/lib/useragent_parser/user_agent.rb

@@ -2,6 +2,7 @@ module UseragentParser
   class UserAgent
     attr_reader :browser, :browser_family, :browser_version, :browser_major_version, :browser_minor_version, :browser_patch_version
     attr_reader :os, :os_family, :os_version, :os_major_version, :os_minor_version, :os_patch_version
+    attr_reader :webmail_client
 
     def initialize(details = {})
       if user_agent = details['user_agent']
@@ -23,6 +24,10 @@ module UseragentParser
          @is_mobile = device['is_mobile']
          @is_spider = device['is_spider']
        end
+
+       if referrer = details['referrer']
+        @webmail_client = referrer['family']
+      end
     end
 
     def device
@@ -53,8 +58,107 @@ module UseragentParser
       @browser ||= browser_name
     end
 
+    def email
+      @email ||= email_name
+    end
+
+    def email_version
+      @email_version ||= email_version_name
+    end
+
+    def is_email?
+      is_outlook? || is_ios_mail? || is_apple_mail? || is_desktop_email? || is_webmail?
+    end
+
     protected
 
+    def is_desktop_email?
+      [
+        "Thunderbird",
+        "T-Online eMail",
+        "Eudora",
+        "Apple Mail",
+        "Sparrow",
+        "Lotus Notes",
+        "Windows Live Mail",
+        "Outlook",
+        "AOL"
+      ].include? @browser_family
+    end
+
+    def is_ios_mail?
+      @webmail_client.nil?  && @browser_family == 'Mobile Safari' && @os_family == 'iOS'
+    end
+
+    def is_apple_mail?
+      @webmail_client.nil? && @browser_family == 'Apple WebKit' && @os_family == 'Mac OS X'
+    end
+
+    def is_webmail?
+      [
+        "Hotmail",
+        "Yahoo! Mail",
+        "WEB.DE",
+        "AOL Webmail",
+        "Gmail",
+        "GMX",
+        "Swisscom",
+        "Strato",
+        "mail.ru"
+      ].include?(@webmail_client)
+    end
+
+    def is_aol?
+      @browser_family == 'AOL' || @webmail_client == 'AOL Webmail'
+    end
+
+    def is_outlook?
+      @browser_family == 'Outlook' || (@browser_family == 'IE' and @webmail_client.nil?)
+    end
+
+    def email_name
+      return 'Microsoft Outlook' if is_outlook?
+      return 'Apple Mail' if is_apple_mail?
+      return 'Apple Mobile Mail' if is_ios_mail?
+      return @browser_family if is_desktop_email?
+      return @webmail_client if is_webmail?
+      return @os_family if @os_family == 'Android'
+      return 'AOL' if is_aol?
+    end
+
+    def email_version_name
+      return apple_mail_names if is_apple_mail?
+      return ios_mail_names if is_ios_mail?
+      return outlook_names if is_outlook?
+      return 'AOL Desktop' if @browser_family == 'AOL'
+      email_name
+    end
+
+    def apple_mail_names
+      case @os_minor_version
+      when '0', '1', '2', '3' then 'Apple Mail 1'
+      when '4' then 'Apple Mail 2'
+      when '5' then 'Apple Mail 3'
+      when '6' then 'Apple Mail 4'
+      when '7' then 'Apple Mail 5'
+      when '8' then 'Apple Mail 6'
+      else 'Apple Mail'
+      end
+    end
+
+    def ios_mail_names
+      @device
+    end
+
+    def outlook_names
+      return 'Outlook 2000/2003/Express' unless @browser_family == 'Outlook'
+      case @browser_major_version
+      when '12' then 'Outlook 2007'
+      when '14' then 'Outlook 2010'
+      else 'Outlook'
+      end
+    end
+
     def browser_name
       "#{browser_family} #{browser_version}"
     end

data/lib/useragent_parser/version.rb

@@ -1,3 +1,3 @@
 module UseragentParser
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/useragent_parser.rb

@@ -2,12 +2,15 @@ require "useragent_parser/version"
 require "useragent_parser/parsers/user_agent_parser"
 require "useragent_parser/parsers/os_parser"
 require "useragent_parser/parsers/device_parser"
+require "useragent_parser/parsers/referrer_parser"
 require "useragent_parser/user_agent"
+require "yaml"
 
 module UseragentParser
   USER_AGENT_PARSERS = []
   OS_PARSERS = []
   DEVICE_PARSERS = []
+  REFERRER_PARSERS = []
   MOBILE_USER_AGENT_FAMILIES = []
   MOBILE_OS_FAMILIES = []
 
@@ -35,12 +38,36 @@ module UseragentParser
       DEVICE_PARSERS.push UseragentParser::DeviceParser.new(regex, device_replacement)
     end
 
+    yaml['referrer_parsers'].each do |parser|
+      regex = parser['regex']
+      referrer_replacement = parser.fetch('referrer_replacement', nil)
+
+      REFERRER_PARSERS.push UseragentParser::ReferrerParser.new(regex, referrer_replacement)
+    end
+
     MOBILE_USER_AGENT_FAMILIES.push *yaml['mobile_user_agent_families']
     MOBILE_OS_FAMILIES.push *yaml['mobile_os_families']
   end
 
-  def self.parse_all(user_agent_string, *js_args)
-    # UseragentParser::UserAgent.new{
+  def self.parse(user_agent_string, *js_args)
+    UseragentParser::UserAgent.new(self.parse_browser(user_agent_string, *js_args))
+  end
+
+  def self.parse_with_referrer(user_agent_string, referrer = nil, *js_args)
+    UseragentParser::UserAgent.new(self.parse_email(user_agent_string, referrer, *js_args))
+  end
+
+  def self.parse_email(user_agent_string, referrer = nil, *js_args)
+    {
+      'user_agent'  => self.parse_user_agent(user_agent_string, *js_args),
+      'os'          => self.parse_os(user_agent_string, *js_args),
+      'device'      => self.parse_device(user_agent_string, *js_args),
+      'string'      => user_agent_string,
+      'referrer'    => self.parse_referrer(referrer)
+    }
+  end
+
+  def self.parse_browser(user_agent_string, *js_args)
     {
       'user_agent'  => self.parse_user_agent(user_agent_string, *js_args),
       'os'          => self.parse_os(user_agent_string, *js_args),
@@ -108,6 +135,16 @@ module UseragentParser
 
     { 'family' => device, 'is_mobile' => is_mobile, 'is_spider' => (device == 'Spider') }
   end
+
+  def self.parse_referrer(referrer_string)
+    referrer = nil
+    REFERRER_PARSERS.each do |parser|
+      referrer = parser.parse(referrer_string)
+      break unless referrer.nil?
+    end
+
+    { 'family' => referrer }
+  end
 end
 
 UseragentParser.load_parsers!

data/spec/fixtures/pgts_browser_list.yaml

@@ -3925,14 +3925,14 @@ test_cases:
       minor:
       patch:
     - user_agent_string: "Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT)"
-      family: "Other"
-      major:
-      minor:
+      family: "Lotus Notes"
+      major: '5'
+      minor: '0'
       patch:
     - user_agent_string: "Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)"
-      family: "Other"
-      major:
-      minor:
+      family: "Lotus Notes"
+      major: '6'
+      minor: '0'
       patch:
     - user_agent_string: "LTH/3.02a (http://www.learntohack.nil)"
       family: "Other"
@@ -5185,13 +5185,13 @@ test_cases:
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/103u (KHTML, like Gecko) safari/100"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '103'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/124 (KHTML, like Gecko)"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '124'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/124 (KHTML, like Gecko, Safari) Shiira/0.9.1"
@@ -5235,8 +5235,8 @@ test_cases:
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/124 (KHTML, like Gecko) safari/125.0"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '124'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.2 (KHTML, like Gecko, Safari) Shiira/0.9.2.2"
@@ -5265,18 +5265,18 @@ test_cases:
       minor: '34'
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/85 (KHTML, like Gecko) OmniWeb/v496"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '85'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/85 (KHTML, like Gecko) OmniWeb/v549"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '85'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/85 (KHTML, like Gecko) OmniWeb/v558"
-      family: "Other"
-      major:
+      family: "Apple WebKit"
+      major: '85'
       minor:
       patch:
     - user_agent_string: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/85 (KHTML, like Gecko) OmniWeb/v558.43"

data/spec/fixtures/test_referrers.yaml

@@ -0,0 +1,40 @@
+test_cases:
+
+  - referrer_string: 'http://bl131w.blu131.mail.live.com/mail/InboxLight.aspx?n=1166653403'
+    family: 'Hotmail'
+
+  - referrer_string: 'http://de.mc250.mail.yahoo.com/mc/welcome?.gx=1&.tm=1290672405&.rand=4rtrlgfb1p8na'
+    family: 'Yahoo! Mail'
+
+  - referrer_string: 'http://email.freenet.de/Email/View/Body?msg=4681&folder=INBOX&showImages=0&showImages=1&updateShowImagesStatus=1'
+    family: 'WEB.DE'
+
+  - referrer_string: 'http://webmail.aol.com/42951/aol/en-us/Suite.aspx'
+    family: 'AOL Webmail'
+
+  - referrer_string: 'http://mail.aol.com/32945-111/aol-1/de-de/Lite/MsgRead.aspx?folder=Spam&uid=1.27991476&seq=1&searchIn=none&searchQuery=&start=0&sort=received'
+    family: 'AOL Webmail'
+
+  - referrer_string: 'http://mail.google.com/mail/?ui=2&view=bsp&ver=ohhl4rw8mbn4'
+    family: 'Gmail'
+
+  - referrer_string: 'http://service.gmx.net/de/cgi/g.fcgi/mail/print?folder=inbox&uid=NDI5OMAkFGxkb%2FUN4WZuvoB2anv6Llsc&CUSTOMERNO=8421009&t=de730856557.1294240410.14bd9d8'
+    family: 'GMX'
+
+  - referrer_string: 'http://www.gmxattachments.net/de/cgi/g.fcgi/mail/print/fullhtml?mid=babgebac.1290790887.16270.a7fshhjuag.74&uid=ZGllQYdlCG9ydask92VvuYdkdFl2iYWj&partUid=ZGllQYdlCG9ydask92VvuYdkdFl2iYWj'
+    family: 'GMX'
+
+  - referrer_string: 'http://proxy-mssazhh.bluewin.ch/mail/MessageRead?sid='
+    family: 'Swisscom'
+
+  - referrer_string: 'http://communicator.strato.de/messages/mail_preview.html?msgNo=964'
+    family: 'Strato'
+
+  - referrer_string: 'http://win.mail.ru/cgi-bin/msglist'
+    family: 'mail.ru'
+
+  - referrer_string: 'http://www.example.com'
+    family: 'Other'
+
+  - referrer_string: ''
+    family: