referer-parser 0.2.2 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWM4OWFlNDZlN2FlOWMwYzhiZmE4YmJmN2I5MjBjYTU1MTJhNmIyZA==
+    YzNmNmNlYjE3ZDdlZGY5M2FjNjAzODFkZGJlNjJkZGJiMzEzOWM0OA==
   data.tar.gz: !binary |-
-    YmYwNTJiZDVkZThkODAwOThmNGNiYTQ2NzM5MmI5NzlmNTRmMWE4MA==
+    MDkyODIyYTdkMjg2ZjYxOGEwNDc3YjcwODE5Zjk2N2Y3YTcxNGNmNw==
 SHA512:
   metadata.gz: !binary |-
-    YzQxYjI1NjA3NmQzYTc1YmEzYjZkYzNhNjMwYjIyOGY4MjU0YzFhZTAzMjc3
-    M2FjY2FmOGIyZDFhZmFjMmViNTkxODA1MDhkMjA1ZjA3MmE0NTZjMjg4OWJi
-    OTllZTdhNTg4NzQyYWYxNjg3MDdhYTczNmU0MzZmZWU5NDFmNTc=
+    NDU5NjM0OGVkMjM3N2M5YTQxMWFkYzU3NTYwZDdjODdkZmJkYmIwMDZlZmUw
+    YzI3NzNhMjllOWU1NTk4Yzg5YjUyMDYzOWM2ZjU0OTBhNDU4YmU0Nzc4YjBk
+    MWI2NzU0ZjNjZjA5ZWNlZjU5M2U3OTU4MGJiMDk4Y2ViMTJiZGQ=
   data.tar.gz: !binary |-
-    ODMzYzdmYjFlNmYyNmQzNjIxMmY0OTQ5ZmU0OTZhZWVjMGQ5YTc2MzQwMWE4
-    ZmJjYTQ1ODcxZWM3MzFlZDE1NGQzZTFlZWVlMGQ3YzNmMjZhMzc3OGU0YmE1
-    YzJlNmJkMzhhN2QxMzdiNjY3Yzk2ZjdjZGFkOGRiNzU5NjQyODc=
+    M2M2YWNhOTc4ODEwMjQxNTdjZTA4YjE5ZDZmMWZjNjJhNGFlMjA0MGRlZjEw
+    NGFkNTQ0MGYyOTlhNWNkYjZlYjhkYzg4NWJlYTU3ZDc1MDRmNjBlM2FkMTEz
+    ZWMzYTdlNWEyYmFkZWU3M2Y5NjI5YThhNDczNGZkMDZmMTk5MmY=

data/README.md

@@ -4,8 +4,6 @@ This is the Ruby implementation of [referer-parser] [referer-parser], the librar
 
 The implementation uses the shared 'database' of known referers found in [`referers.yml`] [referers-yml].
 
-**Currently the Ruby library only extracts search engine referers - it needs updating with the additional functionality now found in the Java/Scala version.**
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -22,20 +20,55 @@ Or install it yourself as:
 
 ## Usage
 
-Use referer-parser like this:
+### To include referer-parser:
 
 ```ruby
 require 'referer-parser'
+```
+
+### To create a parser
+
+Parsers are created by default with the set of included referers but they can also be loaded from another file(s) either during or after instantiation
+
+Creating and modifying the parser:
+
+```ruby
+# Default parser
+parser = RefererParser::Parser.new
 
-referer_url = 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari'
+# Custom parser with local file
+parser = RefererParser::Parser.new('/path/to/other/referers.yml')
 
-r = RefererParser::Referer.new(referer_url)
+# From a URI
+parser = RefererParser::Parser.new('http://example.com/path/to/other/referers.yml')
 
-puts r.known?                # =>  true
-puts r.referer               # => 'Google'
-puts r.search_parameter      # => 'q'     
-puts r.search_term           # => 'gateway oracle cards denise linn'
-puts r.uri.host              # => 'www.google.com'
+# Default referers, then merge in a set of custom internal domains
+parser = RefererParser::Parser.new
+parser.update('/path/to/internal.yml')
+
+# Default referers, then add your own internal domain inline instead of from a file
+parser = RefererParser::Parser.new
+parser.add_referer('internal', 'SnowPlow', 'snowplowanalytics.com')
+
+# Clear all of the existing referers
+parser.clear!
+```
+
+### Using a parser
+
+The parser returns a hash of matching data if it can be found including search terms, medium, and nicely-formatted source name.
+If there is no match, :known will be false.
+
+```ruby
+parser = RefererParser::Parser.new
+parser.parse('http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari')
+  # => {
+    :known=>true,
+    :uri=>"http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari",
+    :source=>"Google",
+    :medium=>"search",
+    :term=>"gateway oracle cards denise linn"
+  }
 ```
 
 ## Contributing
@@ -48,7 +81,7 @@ puts r.uri.host              # => 'www.google.com'
 
 ## Copyright and license
 
-The referer-parser Ruby library is copyright 2012-2013 Snowplow Analytics Ltd.
+The referer-parser Ruby library is copyright 2014 Inside Systems, Inc.
 
 Licensed under the [Apache License, Version 2.0] [license] (the "License");
 you may not use this software except in compliance with the License.
@@ -62,4 +95,4 @@ limitations under the License.
 [referer-parser]: https://github.com/snowplow/referer-parser
 [referers-yml]: https://github.com/snowplow/referer-parser/blob/master/referers.yml
 
-[license]: http://www.apache.org/licenses/LICENSE-2.0
+[license]: http://www.apache.org/licenses/LICENSE-2.0

data/Rakefile

@@ -1 +1,10 @@
 require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec

data/lib/referer-parser.rb

@@ -15,9 +15,7 @@
 
 require "referer-parser/version"
 require "referer-parser/errors"
-require "referer-parser/referers"
-require "referer-parser/referer"
+require "referer-parser/parser"
 
 module RefererParser
-  # Your code goes here...
-end
+end

data/lib/referer-parser/errors.rb

@@ -15,8 +15,6 @@
 
 module RefererParser
 
-  # Errors thrown by RefererParser
-
   class RefererParserError < StandardError
     attr_reader :original
     def initialize(msg, original=nil);
@@ -25,12 +23,7 @@ module RefererParser
     end
   end
 
-  class InvalidUriError < StandardError
-  end
-
-  class ReferersYamlNotFoundError < StandardError
-  end
-
-  class CorruptReferersYamlError < StandardError
-  end
-end
+  class UnsupportedFormatError < RefererParserError; end
+  class InvalidUriError < RefererParserError; end
+  class CorruptReferersError < RefererParserError; end
+end

data/lib/referer-parser/parser.rb

@@ -0,0 +1,215 @@
+# Copyright (c) 2014 Inside Systems, Inc All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Kelley Reynolds (mailto:kelley@insidesystems.net)
+# Copyright:: Copyright (c) 2014 Inside Systems Inc
+# License::   Apache License Version 2.0
+
+require 'uri'
+require 'cgi'
+
+module RefererParser
+  class Parser
+    DefaultFile = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'data', 'referers.json'))
+
+    # Create a new parser from one or more filenames/uris, defaults to ../data/referers.json
+    def initialize(uris=DefaultFile)
+      @domain_index ||= {}
+      @name_hash ||= {}
+
+      update(uris)
+    end
+
+    # Update the referer database with one or more uris
+    def update(uris)
+      [uris].flatten.each do |uri|
+        deserialize_referer_data(read_referer_data(uri), File.extname(uri).downcase)
+      end
+
+      true
+    end
+
+    # Clean out the database
+    def clear!
+      @domain_index, @name_hash = {}, {}
+
+      true
+    end
+
+    # Add a referer to the database with medium, name, domain or array of domains, and a parameter or array of parameters
+    # If called manually and a domain is added to an existing entry with a path, you may need to call optimize_index! afterwards.
+    def add_referer(medium, name, domains, parameters=nil)
+      # The same name can be used with multiple mediums so we make a key here
+      name_key = "#{name}-#{medium}"
+
+      # Update the name has with the parameter and medium data
+      @name_hash[name_key] = {:source => name, :medium => medium, :parameters => [parameters].flatten }
+
+      # Update the domain to name index
+      [domains].flatten.each do |domain_url|
+        domain, *path = domain_url.split('/')
+        if domain =~ /\Awww\.(.*)\z/i
+          domain = $1
+        end
+
+        domain.downcase!
+
+        @domain_index[domain] ||= []
+        if !path.empty?
+          @domain_index[domain] << ['/' + path.join('/'), name_key]
+        else
+          @domain_index[domain] << ['/', name_key]
+        end
+      end
+    end
+
+    # Prune duplicate entries and sort with the most specific path first if there is more than one entry
+    # In this case, sorting by the longest string works fine
+    def optimize_index!
+      @domain_index.each do |key, val|
+        # Sort each path/name_key pair by the longest path
+        @domain_index[key].sort! { |a, b|
+          b[0].size <=> a[0].size
+        }.uniq!
+      end
+    end
+
+    # Given a string or URI, return a hash of data
+    def parse(obj)
+      url = obj.is_a?(URI) ? obj : URI.parse(obj.to_s)
+
+      if !['http', 'https'].include?(url.scheme)
+        raise InvalidUriError.new("Only HTTP and HTTPS schemes are supported -- #{url.scheme}") 
+      end
+
+      data = { :known => false, :uri => url.to_s }
+
+      domain, name_key = domain_and_name_key_for(url)
+      if domain and name_key
+        referer_data = @name_hash[name_key]
+        data[:known] = true
+        data[:source] = referer_data[:source]
+        data[:medium] = referer_data[:medium]
+        data[:domain] = domain
+
+        # Parse parameters if the referer uses them
+        if url.query and referer_data[:parameters]
+          query_params = CGI.parse(url.query)
+          referer_data[:parameters].each do |param|
+            # If there is a matching parameter, get the first non-blank value
+            if !(values = query_params[param]).empty?
+              data[:term] = values.select { |v| v.strip != "" }.first
+              break if data[:term]
+            end
+          end
+        end
+      end
+
+      data
+    rescue URI::InvalidURIError
+      raise InvalidUriError.new("Unable to parse URI, not a URI? -- #{obj.inspect}", $!)
+    end
+
+    protected
+
+    # Determine the correct name_key for this host and path
+    def domain_and_name_key_for(uri)
+      # Create a proc that will return immediately
+      check = Proc.new do |domain|
+        domain.downcase!
+        if paths = @domain_index[domain]
+          paths.each do |path, name_key|
+            return [domain, name_key] if uri.path.include?(path)
+          end
+        end
+      end
+
+      # First check hosts with and without the www prefix with the path
+      if uri.host =~ /\Awww\.(.+)\z/i
+        check.call $1
+      else
+        check.call uri.host
+      end
+
+      # Remove subdomains until only three are left (probably good enough)
+      host_arr = uri.host.split(".")
+      while host_arr.size > 2 do
+        host_arr.shift
+        check.call host_arr.join(".")
+      end
+
+      nil
+    end
+
+    def deserialize_referer_data(data, ext)
+      # Parse the loaded data with the correct parser
+      deserialized_data = if ['.yml', '.yaml'].include?(ext)
+        deserialize_yaml(data)
+      elsif ext == '.json'
+        deserialize_json(data)
+      else
+        raise UnsupportedFormatError.new("Only yaml and json file formats are currently supported -- #{@msg}")
+      end
+
+      begin
+        parse_referer_data deserialized_data
+      rescue
+        raise CorruptReferersError.new("Unable to parse data file -- #{$!.class} #{$!.to_s}", $!)
+      end
+    end
+
+    def deserialize_yaml(data)
+      require 'yaml'
+      YAML.load(data)
+    rescue Exception => e
+      raise CorruptReferersError.new("Unable to YAML file -- #{e.to_s}", e)
+    end
+
+    def deserialize_json(data)
+      require 'json'
+      JSON.parse(data)
+    rescue JSON::ParserError
+      raise CorruptReferersError.new("Unable to JSON file -- #{$!.to_s}", $!)
+    end
+
+    def read_referer_data(uri)
+      # Attempt to read the data from the network if application, or the file on the local system
+      if uri =~ /\A(?:ht|f)tps?:\/\//
+        require 'open-uri'
+        begin
+          open(uri).read
+        rescue OpenURI::HTTPError
+          raise InvalidUriError.new("Cannot load referer data from URI #{uri} -- #{$!.to_s}", $!)
+        end
+      else
+        File.read(uri)
+      end
+    end
+
+    # Create an index that maps domains/paths to their name/medium and a hash that contains their metadata
+    # The index strips leading www in order to keep the index smaller
+    # Format of the domain_index:
+    #   { domain => [[path1, name_key], [path2, name_key], ... ] }
+    # Format of the name_hash:
+    #   { name_key => {:source, :medium, :parameters} }
+    def parse_referer_data(data)
+      data.each do |medium, name_hash|
+        name_hash.each do |name, name_data|
+          add_referer(medium, name, name_data['domains'], name_data['parameters'])
+        end
+      end
+
+      optimize_index!
+    rescue
+      raise CorruptReferersError.new("Unable to parse referer data", $!)
+    end
+  end
+end

data/lib/referer-parser/version.rb

@@ -14,6 +14,6 @@
 # License::   Apache License Version 2.0
 
 module RefererParser
-	NAME = "referer-parser"
-	VERSION = "0.2.2"
+  NAME = "referer-parser"
+  VERSION = "0.3.0"
 end

data/referer-parser.gemspec

@@ -19,7 +19,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'referer-parser/version'
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Yali Sassoon", "Martin Loy", "Alex Dean"]
+  gem.authors       = ["Yali Sassoon", "Martin Loy", "Alex Dean", "Kelley Reynolds"]
   gem.email         = ["support@snowplowanalytics.com"]
   gem.description   = %q{Library for extracting marketing attribution data from referer URLs}
   gem.summary       = %q{Library for extracting marketing attribution data (e.g. search terms) from referer (sic) URLs. This is used by Snowplow (http://github.com/snowplow/snowplow). Our hope is that this library (and referers.yml) will be extended by anyone interested in parsing referer URLs.}
@@ -34,4 +34,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rspec", "~> 2.6"
+  gem.add_development_dependency "rake", ">= 0.9.2"
 end

data/spec/fixtures/internal.json

@@ -0,0 +1,9 @@
+{
+  "internal": {
+    "SnowPlow": {
+      "domains": [
+        "www.snowplowanalytics.com"
+      ]
+    }
+  }
+}

data/spec/fixtures/invalid.json

@@ -0,0 +1 @@
+This has the right extension but is unparsable gibberish to json:{}}}}

data/spec/fixtures/invalid.yml

@@ -0,0 +1,2 @@
+this:is invalid:yaml:
+!!!

data/spec/{referer-tests.json → fixtures/referer-tests.json}

@@ -121,7 +121,7 @@
     },
 	{
         "spec": "Ask toolbar search #2",
-        "uri": "http://search.tb.ask.com/search/GGmain.jhtml?&st=hp&p2=^ZU^xdm458^YYA^us&n=77fda1bd&ptb=F0B68CA5-4791-4376-BFCC-5F0100329FB6&si=CMKg9-nX07oCFSjZQgodcikACQ&tpr=hpsbsug&searchfor=test",
+        "uri": "http://search.tb.ask.com/search/GGmain.jhtml?&st=hp&p2=%5EZU%5Exdm458%5EYYA%5Eus&n=77fda1bd&ptb=F0B68CA5-4791-4376-BFCC-5F0100329FB6&si=CMKg9-nX07oCFSjZQgodcikACQ&tpr=hpsbsug&searchfor=test",
         "medium": "search",
         "source": "Ask Toolbar",
         "term": "test",
@@ -129,7 +129,7 @@
     },
     {
         "spec": "Voila search",
-        "uri": "http://lemoteur.ke.voila.fr/?module=voila&bhv=web_fr&kw=test",
+        "uri": "http://search.ke.voila.fr/?module=voila&bhv=web_fr&kw=test",
         "medium": "search",
         "source": "Voila",
         "term": "test",
@@ -219,16 +219,16 @@
         "spec": "Internal HTTP",
         "uri": "http://www.snowplowanalytics.com/about/team",
         "medium": "internal",
-        "source": null,
+        "source": "SnowPlow",
         "term": null,
-        "known": false
+        "known": true
     },
     {
         "spec": "Internal HTTPS",
         "uri": "https://www.snowplowanalytics.com/account/profile",
         "medium": "internal",
-        "source": null,
+        "source": "SnowPlow",
         "term": null,
-        "known": false
+        "known": true
     }
 ]

data/spec/parser_spec.rb

@@ -0,0 +1,181 @@
+# Copyright (c) 2014 Inside Systems, Inc All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Kelley Reynolds (mailto:kelley@insidesystems.net)
+# Copyright:: Copyright (c) 2014 Inside Systems, Inc
+# License::   Apache License Version 2.0
+
+require 'spec_helper'
+
+describe RefererParser::Parser do
+  let(:remote_file) { "https://raw.githubusercontent.com/snowplow/referer-parser/master/ruby/data/referers.json" }
+  let(:default_parser) { RefererParser::Parser.new }
+  let(:internal_parser) { RefererParser::Parser.new(fixture('internal.json')) }
+  let(:combined_parser) { RefererParser::Parser.new([RefererParser::Parser::DefaultFile, fixture('internal.json')]) }
+  let(:remote_parser) { RefererParser::Parser.new(remote_file) }
+  let(:domain_index) { parser.instance_variable_get(:@domain_index) }
+  let(:name_hash) { parser.instance_variable_get(:@name_hash) }
+
+  # This gets overridden for different parsers in subsections
+  let(:parser) { default_parser }
+
+  describe "exceptions" do
+    it "should raise UnsupportedFormatError" do
+      lambda { parser.update(__FILE__) }.should raise_error(RefererParser::UnsupportedFormatError)
+    end
+
+    it "should raise CorruptReferersError with invalid json" do
+      lambda { parser.update(fixture('invalid.json')) }.should raise_error(RefererParser::CorruptReferersError)
+    end
+
+    it "should raise CorruptReferersError with invalid yaml" do
+      lambda { parser.update(fixture('invalid.yml')) }.should raise_error(RefererParser::CorruptReferersError)
+    end
+
+    it "should raise CorruptReferersError with valid file with invalid data" do
+      lambda { parser.update(fixture('referer-tests.json')) }.should raise_error(RefererParser::CorruptReferersError)
+    end
+
+    it "should raise InvalidUriError with insane" do
+      lambda { parser.parse('>total gibberish<') }.should raise_error(RefererParser::InvalidUriError)
+    end
+
+    it "should raise InvalidUriError with non http(s)" do
+      lambda { parser.parse('ftp://ftp.really.com/whatever.json') }.should raise_error(RefererParser::InvalidUriError)
+    end
+  end
+
+  describe "with the default parser" do
+    it "should have a non-empty domain_index" do
+      domain_index.should_not be_empty
+    end
+
+    it "should have a non-empty name_hash" do
+      name_hash.should_not be_empty
+    end
+
+    it "should be clearable" do
+      parser.clear!
+      name_hash.should be_empty
+      domain_index.should be_empty
+    end
+
+    it "should be updatable" do
+      size = domain_index.size
+      parser.update(fixture('internal.json'))
+      domain_index.size.should > size
+    end
+  end
+
+  describe "with the internal parser" do
+    let(:parser) { internal_parser }
+
+    it "should have internal mediums only" do
+      domain_index.each_value do |(arr)|
+        path, name_key = arr[0], arr[1]
+        name_hash[name_key][:medium].should == 'internal'
+      end
+    end
+  end
+
+  describe "with the remote parser" do
+    let(:parser) { remote_parser }
+
+    # These are combined here to reduce network fetches
+    it "should have a non-empty domain_index and name_hash" do
+      domain_index.should_not be_empty
+      name_hash.should_not be_empty
+    end
+  end
+
+  describe "sample fixtures" do
+    let(:parser) { combined_parser }
+    # Include our internal data as well
+    JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'referer-tests.json'))).each do |fixture|
+      it fixture['spec'] do
+        parsed_as_string, parsed_as_uri = nil, nil
+        lambda { parsed_as_string = parser.parse(fixture['uri']) }.should_not raise_error
+        lambda { parsed_as_uri = parser.parse(URI.parse(fixture['uri'])) }.should_not raise_error
+
+        ['source', 'term', 'known', 'medium'].each do |key|
+          parsed_as_uri[key.to_sym].should == fixture[key]
+          parsed_as_string[key.to_sym].should == fixture[key]
+        end
+      end
+    end
+  end
+  
+  describe "general behavior" do
+    it "should return the better result when the referer contains two or more parameters" do
+      parsed = parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&q=&key=hello")
+      parsed[:term].should == "hello"
+    end
+
+    it "should return the better result when the referer contains same parameters" do
+      parsed = parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&key=&key=hello")
+      parsed[:term].should == "hello"
+    end
+
+    it "should return the normalized domain" do
+      parsed = parser.parse("http://it.images.search.YAHOO.COM/images/view;_ylt=A0PDodgQmGBQpn4AWQgdDQx.;_ylu=X3oDMTBlMTQ4cGxyBHNlYwNzcgRzbGsDaW1n?back=http%3A%2F%2Fit.images.search.yahoo.com%2Fsearch%2Fimages%3Fp%3DEarth%2BMagic%2BOracle%2BCards%26fr%3Dmcafee%26fr2%3Dpiv-web%26tab%3Dorganic%26ri%3D5&w=1064&h=1551&imgurl=mdm.pbzstatic.com%2Foracles%2Fearth-magic-oracle-cards%2Fcard-1.png&rurl=http%3A%2F%2Fwww.psychicbazaar.com%2Foracles%2F143-earth-magic-oracle-cards.html&size=2.8+KB&name=Earth+Magic+Oracle+Cards+-+Psychic+Bazaar&p=Earth+Magic+Oracle+Cards&oid=f0a5ad5c4211efe1c07515f56cf5a78e&fr2=piv-web&fr=mcafee&tt=Earth%2BMagic%2BOracle%2BCards%2B-%2BPsychic%2BBazaar&b=0&ni=90&no=5&ts=&tab=organic&sigr=126n355ib&sigb=13hbudmkc&sigi=11ta8f0gd&.crumb=IZBOU1c0UHU")
+      parsed[:domain].should == "images.search.yahoo.com"
+    end
+  end
+
+  describe "optimize_index" do
+    let(:domains) { ['fnord.com', 'fnord.com', 'fnord.com/path'] }
+
+    before do
+      parser.add_referer('internal', 'Fnord', domains)
+    end
+
+    it "should have out of order and duplicate domains before optimization" do
+      domain_index['fnord.com'].transpose.first.should == ['/', '/', '/path']
+    end
+
+    it "should have out of order domains before optimization" do
+      parser.optimize_index!
+      domain_index['fnord.com'].transpose.first.should == ['/path', '/']
+    end
+  end
+
+  describe "add_referer" do
+    it "should add a referer to the domain_index" do
+      domain_index['fnord.com'].should be_nil
+      parser.add_referer('internal', 'Fnord', 'fnord.com')
+      domain_index['fnord.com'].should_not be_nil
+    end
+
+    it "should add a referer with multiple domains to the domain_index" do
+      domain_index['fnord.com'].should be_nil
+      domain_index['boo.com'].should be_nil
+      parser.add_referer('internal', 'Fnord', ['fnord.com', 'boo.com'])
+      domain_index['fnord.com'].should_not be_nil
+      domain_index['boo.com'].should_not be_nil
+    end
+
+    it "should add a referer to the name_hash" do
+      name_hash['fnord.com-internal'].should be_nil
+      parser.add_referer('internal', 'Fnord', 'fnord.com')
+      name_hash['Fnord-internal'].should_not be_nil
+    end
+
+    it "should add parameters to the name_hash" do
+      parser.add_referer('internal', 'Fnord', 'fnord.com', ['Q', 'q'])
+      name_hash['Fnord-internal'][:parameters].should == ['Q', 'q']
+    end
+
+    it "should add a single parameter to the name_hash" do
+      parser.add_referer('internal', 'Fnord', 'fnord.com', 'q')
+      name_hash['Fnord-internal'][:parameters].should == ['q']
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+# Copyright (c) 2014 Inside Systems, Inc All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Kelley Reynolds (mailto:kelley@insidesystems.net)
+# Copyright:: Copyright (c) 2014 Inside Systems, Inc
+# License::   Apache License Version 2.0
+
+
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default, :test)
+
+require 'yaml'
+require 'rspec'
+require 'referer-parser'
+require 'uri'
+require 'json'
+
+module Helpers
+  def fixture(filename)
+    File.join(File.dirname(__FILE__), 'fixtures', filename)
+  end
+end
+
+RSpec.configure do |config|
+  config.include Helpers
+end

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: referer-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Yali Sassoon
 - Martin Loy
 - Alex Dean
+- Kelley Reynolds
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -26,6 +27,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
 description: Library for extracting marketing attribution data from referer URLs
 email:
 - support@snowplowanalytics.com
@@ -42,12 +57,15 @@ files:
 - data/referers.yml
 - lib/referer-parser.rb
 - lib/referer-parser/errors.rb
-- lib/referer-parser/referer.rb
-- lib/referer-parser/referers.rb
+- lib/referer-parser/parser.rb
 - lib/referer-parser/version.rb
 - referer-parser.gemspec
-- spec/referer-spec.rb
-- spec/referer-tests.json
+- spec/fixtures/internal.json
+- spec/fixtures/invalid.json
+- spec/fixtures/invalid.yml
+- spec/fixtures/referer-tests.json
+- spec/parser_spec.rb
+- spec/spec_helper.rb
 homepage: http://github.com/snowplow/referer-parser
 licenses: []
 metadata: {}
@@ -67,7 +85,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.1.11
 signing_key: 
 specification_version: 4
 summary: Library for extracting marketing attribution data (e.g. search terms) from
@@ -75,5 +93,9 @@ summary: Library for extracting marketing attribution data (e.g. search terms) f
   Our hope is that this library (and referers.yml) will be extended by anyone interested
   in parsing referer URLs.
 test_files:
-- spec/referer-spec.rb
-- spec/referer-tests.json
+- spec/fixtures/internal.json
+- spec/fixtures/invalid.json
+- spec/fixtures/invalid.yml
+- spec/fixtures/referer-tests.json
+- spec/parser_spec.rb
+- spec/spec_helper.rb

data/lib/referer-parser/referer.rb

@@ -1,118 +0,0 @@
-# Copyright (c) 2012-2013 Snowplow Analytics Ltd. All rights reserved.
-#
-# This program is licensed to you under the Apache License Version 2.0,
-# and you may not use this file except in compliance with the Apache License Version 2.0.
-# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the Apache License Version 2.0 is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
-
-# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012-2013 Snowplow Analytics Ltd
-# License::   Apache License Version 2.0
-
-require 'uri'
-require 'cgi'
-
-module RefererParser
-  class Referer
-
-    attr_reader :uri,
-                :known,
-                :referer,
-                :search_parameter,
-                :search_term
-
-    # So can be interrogated with .known? too.
-    alias_method :known?, :known
-
-    def parse(referer_url)   
-      @uri = Referer::parse_uri(referer_url)
-
-      referer = Referers::get_referer(@uri)
-      unless referer.nil?
-        @known = true
-        @referer = referer['name']
-        @search_parameter, @search_term = Referer::extract_search(@uri, referer['parameters'])
-      else
-        @known = false
-        @referer, @search_parameter, @search_term = nil # Being explicit
-      end
-    end
-
-    private # -------------------------------------------------------------
-
-    # Static method to turn a `raw_url`
-    # into a URI, checking that it's
-    # a HTTP(S) URI. Supports raw
-    # string and existing URI
-    def self.parse_uri(raw_url)
-
-      uri = if raw_url.is_a? String
-              begin
-                URI.parse(raw_url)
-              rescue => error
-                raise InvalidUriError, error.message
-              end
-            elsif raw_url.is_a? URI
-              raw_url
-            else
-              raise InvalidUriError, "'#{raw_url}' must be a String or URI"
-            end
-
-      unless %w( http https ).include?(uri.scheme)
-        raise InvalidUriError, "'#{raw_url}' is not an http(s) protocol URI"
-      end
-      uri
-    end
-
-    # Static method to get the keywords from a `uri`,
-    # where keywords are stored against one of the
-    # `possible_parameters` in the querystring.
-    # Returns a 'tuple' of the parameter found plus
-    # the keywords.
-    def self.extract_search(uri, possible_parameters)
-      param = nil
-
-      # Only get keywords if there's a query string to extract them from...
-      if uri.query
-        parameters = CGI.parse(uri.query)
-
-        # Try each possible keyword parameter with the querystring until one returns a result
-        possible_parameters.each do | pp |
-          if parameters.has_key?(pp)
-            param = pp
-            parameters[pp].each do |result|
-              unless result == ""
-                return [pp, result] # return first value not eql ""
-              end
-            end
-          end
-        end
-      end
-
-      return [param, []] # No parameter or keywords to return
-    end
-  
-    # Constructor. Takes the `referer_url`
-    # to extract the referer from (can be
-    # a String or URI)
-    #
-    # Optionaly it takes the `referer_file` param
-    # to use instead of the bundle referers.yml
-    # (must be a yaml file)
-    def initialize(referer_url, referer_file = nil)
-      
-      if referer_file.nil?
-        Referers::load_referers_from_yaml(Referers::get_yaml_file())
-      else
-        Referers::load_referers_from_yaml(Referers::get_yaml_file(referer_file))
-      end
-
-      parse(referer_url)
-      
-    end
-  end
-end

data/lib/referer-parser/referers.rb

@@ -1,92 +0,0 @@
-# Copyright (c) 2012-2013 Snowplow Analytics Ltd. All rights reserved.
-#
-# This program is licensed to you under the Apache License Version 2.0,
-# and you may not use this file except in compliance with the Apache License Version 2.0.
-# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the Apache License Version 2.0 is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
-
-# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012-2013 Snowplow Analytics Ltd
-# License::   Apache License Version 2.0
-
-require 'yaml'
-
-# This module processes the referers.yml file and
-# uses it to create a global hash that is used to
-# lookup URLs to see if they are known referers
-# (e.g. search engines)
-module RefererParser
-  module Referers
-
-    # Returns the referer indicated by
-    # the given `uri`
-    def self.get_referer(uri)
-      # Check if domain+path matches (e.g. google.co.uk/products)
-      referer = @referers[uri.host + uri.path]
-      if referer.nil?
-        # Check if domain only matches (e.g. google.co.uk)
-        referer = @referers[uri.host]
-      end
-      referer
-    end
-
-    private # -------------------------------------------------------------
-    
-    # Returns the path to the YAML
-    # file of referers
-    def self.get_yaml_file(referer_file = nil)
-      if referer_file.nil?
-        File.join(File.dirname(__FILE__), '..', '..', 'data', 'referers.yml')
-      else
-        referer_file
-      end
-    end
-
-    # Initializes a hash of referers
-    # from the supplied YAML file
-    def self.load_referers_from_yaml(yaml_file)
-      return if @loaded_file == yaml_file
-      unless File.exist?(yaml_file) and File.file?(yaml_file)
-        raise ReferersYamlNotFoundError, "Could not find referers YAML file at '#{yaml_file}'"
-      end
-
-      # Load referer data stored in YAML file
-      begin
-        yaml = YAML.load_file(yaml_file)['search'] # TODO: fix this when we support the other types
-      rescue error
-        raise CorruptReferersYamlError.new("Could not parse referers YAML file '#{yaml_file}'", error)
-      end
-      @referers = load_referers(yaml)
-      @loaded_file = yaml_file
-    end
-
-    # Validate and expand the `raw_referers`
-    # array, building a hash of referers as
-    # we go
-    def self.load_referers(raw_referers)
-
-      # Validate the YAML file, building the lookup
-      # hash of referer domains as we go
-      referers = Hash.new
-      raw_referers.each { | referer, data |
-        if data['parameters'].nil?
-          raise CorruptReferersYamlError, "No parameters found for referer '#{referer}'"
-        end
-        if data['domains'].nil? 
-          raise CorruptReferersYamlError, "No domains found for referer '#{referer}'"
-        end
-        
-        data['domains'].each do | domain |
-          domain_pair = { domain => { "name" => referer,
-                                      "parameters" => data['parameters']}}
-          referers.merge!(domain_pair)
-        end
-      }
-      return referers 
-    end
-  end
-end

data/spec/referer-spec.rb

@@ -1,92 +0,0 @@
-# Copyright (c) 2012-2013 Snowplow Analytics Ltd. All rights reserved.
-#
-# This program is licensed to you under the Apache License Version 2.0,
-# and you may not use this file except in compliance with the Apache License Version 2.0.
-# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the Apache License Version 2.0 is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
-
-# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012-2013 Snowplow Analytics Ltd
-# License::   Apache License Version 2.0
-
-require 'referer-parser'
-require 'uri'
-
-describe RefererParser::Referer do
-
-  GOOGLE_COM_REFERER   = 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari&tbo=d&biw=768&bih=900&source=lnms&tbm=isch&ei=t9fTT_TFEYb28gTtg9HZAw&sa=X&oi=mode_link&ct=mode&cd=2&sqi=2&ved=0CEUQ_AUoAQ'
-  GOOGLE_CO_UK_REFERER = 'http://www.google.co.uk/search?hl=en&client=safari&q=psychic+bazaar&oq=psychic+bazaa&aq=0&aqi=g1&aql=&gs_l=mobile-gws-serp.1.0.0.61498.64599.0.66559.12.9.1.1.2.2.2407.10525.6-2j0j1j3.6.0...0.0.DiYO_7K_ndg&mvs=0'
-  FACEBOOK_COM_REFERER = 'http://www.facebook.com/l.php?u=http%3A%2F%2Fpsy.bz%2FLtPadV&h=MAQHYFyRRAQFzmokHhn3w4LGWVzjs7YwZGejw7Up5TqNHIw'
-  TRUNCATED_REFERER = 'http://googleads.g.doubleclick.net/pagead/ads?client=ca-pub-9108147844898389&output=html&h=60&slotname=1720218904&w=468&lmt=1368485108&flash=11.7.700.169&url=http%3A%2F%2Fwww.bsaving.com%2Fprintable-online-target-coupons%3Futm_source%3Dbsaving_new_Email%2'
-
-  it "Should be initializable with an external referers.yml" do
-    external_referer = File.join(File.dirname(__FILE__), '..', 'data', 'referers.yml') # Using the bundled referers.yml in fact
-    uri = URI.parse(GOOGLE_COM_REFERER)
-    r = RefererParser::Referer.new(uri, external_referer)
-    r.referer.should eql "Google" 
-  end
-  
-  it "Should be initializable without an external referers.yml" do
-    uri = URI.parse(GOOGLE_COM_REFERER)
-    r = RefererParser::Referer.new(uri)
-    r.referer.should eql "Google"   
-  end
-  
-  it "Should correctly parse a google.com referer URL" do
-    r = RefererParser::Referer.new(GOOGLE_COM_REFERER)
-    r.known?.should eql true
-    r.referer.should eql "Google"
-    r.search_parameter.should eql "q"
-    r.search_term.should eql "gateway oracle cards denise linn"
-    r.uri.host.should eql "www.google.com"
-  end
-
-  it "Should correctly extract a google.co.uk search term" do
-    r = RefererParser::Referer.new(GOOGLE_CO_UK_REFERER)
-    r.search_term.should eql "psychic bazaar"
-  end
-
-  it "Should not identify Facebook as a known referer" do
-    r = RefererParser::Referer.new(FACEBOOK_COM_REFERER)
-    r.known?.should eql false
-  end
-
-  it "Should be initializable with an existing URI object" do
-    uri = URI.parse(GOOGLE_COM_REFERER)
-    r = RefererParser::Referer.new(uri)
-    r.referer.should eql "Google"
-  end
-
-  it "Should be possible to re-use a Referer object" do
-    r = RefererParser::Referer.new(GOOGLE_CO_UK_REFERER)
-    r.search_term.should eql "psychic bazaar"
-    r.parse(GOOGLE_COM_REFERER)
-    r.search_term.should eql "gateway oracle cards denise linn"
-    r.uri.host.should eql "www.google.com"
-  end
-
-  it "Should return the better result when the referer contains two or more parameters" do
-    referer_contains_two_params = "http://search.tiscali.it/?tiscalitype=web&collection=web&q=&key=hello"
-    r = RefererParser::Referer.new(referer_contains_two_params)
-    r.search_term.should eql "hello"
-    r.search_parameter.should eql "key"
-  end
-
-  it "Should return the better result when the referer contains same parameters" do
-    referer_contains_two_params = "http://search.tiscali.it/?tiscalitype=web&collection=web&key=&key=hello"
-    r = RefererParser::Referer.new(referer_contains_two_params)
-    r.search_term.should eql "hello"
-    r.search_parameter.should eql "key"
-  end
-
-  it "should raise InvalidUriError on a truncated Uri" do
-    expect{
-      r = RefererParser::Referer.new(TRUNCATED_REFERER)
-    }.to raise_error(RefererParser::InvalidUriError)
-  end
-
-end