referer-parser 0.0.3

data/lib/referer-parser/errors.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+module RefererParser
+
+  # Errors thrown by RefererParser
+
+  class RefererParserError < StandardError
+    attr_reader :original
+    def initialize(msg, original=nil);
+      super(msg);
+      @original = original;
+    end
+  end
+
+  class InvalidUriError < StandardError
+  end
+
+  class ReferersYamlNotFoundError < StandardError
+  end
+
+  class CorruptReferersYamlError < StandardError
+  end
+end

data/lib/referer-parser/referer.rb

@@ -0,0 +1,97 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'uri'
+require 'cgi'
+
+module RefererParser
+  class Referer
+
+    attr_reader :uri,
+                :known,
+                :referer,
+                :search_parameter,
+                :search_term
+
+    # So can be interrogated with .known? too.
+    alias_method :known?, :known
+
+    private # -------------------------------------------------------------
+
+    # Static method to turn a `raw_url`
+    # into a URI, checking that it's
+    # a HTTP(S) URI. Supports raw
+    # string and existing URI
+    def self.parse_uri(raw_url)
+
+      uri = if raw_url.is_a? String
+              begin
+                URI.parse(raw_url)
+              rescue error
+                raise InvalidUriError.new("Cannot parse String #{raw_url} into URI", error)
+              end
+            elsif raw_url.is_a? URI
+              raw_url
+            else
+              raise InvalidUriError "'#{raw_url}' must be a String or URI"
+            end
+
+      unless %w( http https ).include?(uri.scheme)
+        raise InvalidUriError, "'#{raw_url}' is not an http(s) protocol URI"
+      end
+      uri
+    end
+
+    # Static method to get the keywords from a `uri`,
+    # where keywords are stored against one of the
+    # `possible_parameters` in the querystring.
+    # Returns a 'tuple' of the parameter found plus
+    # the keywords.
+    def self.extract_search(uri, possible_parameters)
+
+      # Only get keywords if there's a query string to extract them from...
+      if uri.query
+        parameters = CGI.parse(uri.query)
+
+        # Try each possible keyword parameter with the querystring until one returns a result
+        possible_parameters.each do | pp |
+          if parameters.has_key?(pp)
+            return [pp, parameters[pp].first] # Silently swallow a second or third value in the array
+          end
+        end
+      end
+
+      return [nil, []] # No parameter or keywords to return
+    end
+
+    # Constructor. Takes the `referer_url`
+    # to extract the referer from (can be
+    # a String or URI)
+    def initialize(referer_url)
+
+      @uri = Referer::parse_uri(referer_url)
+
+      referer = Referers::get_referer(@uri)
+      unless referer.nil?
+        @known = true
+        @referer = referer['name']
+        @search_parameter, @search_term = Referer::extract_search(@uri, referer['parameters'])
+      else
+        @known = false
+        @referer, @search_parameter, @search_term = nil # Being explicit
+      end
+    end
+  end
+end

data/lib/referer-parser/referers.rb

@@ -0,0 +1,89 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'yaml'
+
+# This module processes the referers.yml file and
+# uses it to create a global hash that is used to
+# lookup URLs to see if they are known referers
+# (e.g. search engines)
+module RefererParser
+  module Referers
+
+    # Returns the referer indicated by
+    # the given `uri`
+    def self.get_referer(uri)
+      # Check if domain+path matches (e.g. google.co.uk/products)
+      referer = @referers[uri.host + uri.path]
+      if referer.nil?
+        # Check if domain only matches (e.g. google.co.uk)
+        referer = @referers[uri.host]
+      end
+      referer
+    end
+
+    private # -------------------------------------------------------------
+    
+    # Returns the path to the YAML
+    # file of referers
+    def self.get_yaml_file
+      File.join(File.dirname(__FILE__), '..', '..', 'data', 'search.yml')
+    end
+
+    # Initializes a hash of referers
+    # from the supplied YAML file
+    def self.load_referers_from_yaml(yaml_file)
+      
+      unless File.exist?(yaml_file) and File.file?(yaml_file)
+        raise ReferersYamlNotFoundError, "Could not find referers YAML file at '#{yaml_file}'"
+      end
+
+      # Load referer data stored in YAML file
+      begin
+        yaml = YAML.load_file(yaml_file)
+      rescue error
+        raise CorruptReferersYamlError.new("Could not parse referers YAML file '#{yaml_file}'", error)
+      end
+      load_referers(yaml)
+    end
+
+    # Validate and expand the `raw_referers`
+    # array, building a hash of referers as
+    # we go
+    def self.load_referers(raw_referers)
+
+      # Validate the YAML file, building the lookup
+      # hash of referer domains as we go
+      referers = Hash.new
+      raw_referers.each { | referer, data |
+        if data['parameters'].nil?
+          raise CorruptReferersYamlError, "No parameters found for referer '#{referer}'"
+        end
+        if data['domains'].nil? 
+          raise CorruptReferersYamlError, "No domains found for referer '#{referer}'"
+        end
+        
+        data['domains'].each do | domain |
+          domain_pair = { domain => { "name" => referer,
+                                      "parameters" => data['parameters']}}
+          referers.merge!(domain_pair)
+        end
+      }
+      return referers 
+    end
+
+    @referers = load_referers_from_yaml(get_yaml_file())
+  end
+end

data/lib/referer-parser/version.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+module RefererParser
+	NAME = "referer-parser"
+	VERSION = "0.0.3"
+end

data/lib/referer-parser.rb

@@ -0,0 +1,23 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require "referer-parser/version"
+require "referer-parser/errors"
+require "referer-parser/referers"
+require "referer-parser/referer"
+
+module RefererParser
+  # Your code goes here...
+end

data/referer-parser.gemspec

@@ -0,0 +1,37 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'referer-parser/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Yali Sassoon"]
+  gem.email         = ["support@snowplowanalytics.com"]
+  gem.description   = %q{Library for extracting marketing attribution data from referer URLs}
+  gem.summary       = %q{Library for extracting marketing attribution data (e.g. search terms) from referer URLs. This is used by SnowPlow (http://github.com/snowplow/snowplow). Our hope is that this library (and referers.yml) will be extended by anyone interested in parsing referer URLs.}
+  gem.homepage      = "http://github.com/snowplow/referer-parser"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = RefererParser::NAME
+  gem.version       = RefererParser::VERSION
+  gem.platform      = Gem::Platform::RUBY
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rspec", "~> 2.6"
+end

data/spec/referer-spec.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'referer-parser'
+require 'uri'
+
+describe RefererParser::Referer do
+
+  GOOGLE_COM_REFERER   = 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari&tbo=d&biw=768&bih=900&source=lnms&tbm=isch&ei=t9fTT_TFEYb28gTtg9HZAw&sa=X&oi=mode_link&ct=mode&cd=2&sqi=2&ved=0CEUQ_AUoAQ'
+  GOOGLE_CO_UK_REFERER = 'http://www.google.co.uk/search?hl=en&client=safari&q=psychic+bazaar&oq=psychic+bazaa&aq=0&aqi=g1&aql=&gs_l=mobile-gws-serp.1.0.0.61498.64599.0.66559.12.9.1.1.2.2.2407.10525.6-2j0j1j3.6.0...0.0.DiYO_7K_ndg&mvs=0'
+  FACEBOOK_COM_REFERER = 'http://www.facebook.com/l.php?u=http%3A%2F%2Fpsy.bz%2FLtPadV&h=MAQHYFyRRAQFzmokHhn3w4LGWVzjs7YwZGejw7Up5TqNHIw'
+
+  it "Should correctly parse a google.com referer URL" do
+    r = RefererParser::Referer.new(GOOGLE_COM_REFERER)
+    r.known?.should eql true
+    r.referer.should eql "Google"
+    r.search_parameter.should eql "q"
+    r.search_term.should eql "gateway oracle cards denise linn"
+    r.uri.host.should eql "www.google.com"
+  end
+
+  it "Should correctly extract a google.co.uk search term" do
+    r = RefererParser::Referer.new(GOOGLE_CO_UK_REFERER)
+    r.search_term.should eql "psychic bazaar"
+  end
+
+  it "Should not identify Facebook as a known referer" do
+    r = RefererParser::Referer.new(FACEBOOK_COM_REFERER)
+    r.known?.should eql false
+  end
+
+  it "Should be initializable with an existing URI object" do
+    uri = URI.parse(GOOGLE_COM_REFERER)
+    r = RefererParser::Referer.new(uri)
+    r.referer.should eql "Google"
+  end
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: referer-parser
+version: !ruby/object:Gem::Version 
+  hash: 25
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 3
+  version: 0.0.3
+platform: ruby
+authors: 
+- Yali Sassoon
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 6
+        version: "2.6"
+  type: :development
+  version_requirements: *id001
+description: Library for extracting marketing attribution data from referer URLs
+email: 
+- support@snowplowanalytics.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE-2.0.txt
+- README.md
+- Rakefile
+- data/search.yml
+- lib/referer-parser.rb
+- lib/referer-parser/errors.rb
+- lib/referer-parser/referer.rb
+- lib/referer-parser/referers.rb
+- lib/referer-parser/version.rb
+- referer-parser.gemspec
+- spec/referer-spec.rb
+homepage: http://github.com/snowplow/referer-parser
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Library for extracting marketing attribution data (e.g. search terms) from referer URLs. This is used by SnowPlow (http://github.com/snowplow/snowplow). Our hope is that this library (and referers.yml) will be extended by anyone interested in parsing referer URLs.
+test_files: 
+- spec/referer-spec.rb