attlib 0.0.1

data/lib/attlib/referrer.rb

@@ -0,0 +1,81 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'attlib/search_engine_lookup'
+require 'uri'
+require 'cgi'
+
+class Referrer
+
+	attr_reader :referrer_url, :search_engine, :possible_keyword_parameters, :keywords
+
+	def initialize(referrer_url)
+		# Check if the URI is valid
+		if uri?(referrer_url)
+			@referrer_url = URI(referrer_url)
+
+			# Check if the referrer is a search engine and if so, assign the values to :search_engine and :keywords
+			
+			# First check if the domain + path matches (e.g. google.co.uk/products) any of the search engines in the lookup hash
+			if $SEARCH_ENGINE_LOOKUP[@referrer_url.host + @referrer_url.path]
+				@search_engine = $SEARCH_ENGINE_LOOKUP[@referrer_url.host + @referrer_url.path]['name']
+				@possible_keyword_parameters = $SEARCH_ENGINE_LOOKUP[@referrer_url.host + @referrer_url.path]['parameters']
+				@keywords = get_keywords
+			
+			# If not, check if the domain by itself matches (e.g. google.co.uk)
+			elsif $SEARCH_ENGINE_LOOKUP[@referrer_url.host]
+				@search_engine = $SEARCH_ENGINE_LOOKUP[@referrer_url.host]['name']
+				@possible_keyword_parameters = $SEARCH_ENGINE_LOOKUP[@referrer_url.host]['parameters']
+				@keywords = get_keywords
+			
+			# Otherwise referrer is not a search engine
+			else
+				@search_engine = nil
+				@possible_keyword_parameters = nil
+				@keywords = nil
+			end
+		else
+			# Otherwise the URI is not valid
+			raise ArgumentError, "#{referrer_url} is not a valid URL"
+		end
+	end
+
+	def is_search_engine?
+		@search_engine
+	end
+
+	def get_keywords
+		# only get keywords if there's a query string to extract them from...
+		if @referrer_url.query
+			query_parameters = CGI.parse(@referrer_url.query)
+
+			# try each possible keyword parameter with the query string until one returns a result
+			possible_keyword_parameters.each do | parameter |
+				if query_parameters.has_key?(parameter)
+					return query_parameters[parameter].to_s
+				end
+			end
+		end
+	end
+
+	def uri?(string)
+		uri = URI.parse(string)
+		%w( http https ).include?(uri.scheme)
+	rescue URI::BadURIError
+		false
+	rescue URI::InvalidURIError
+		false
+	end
+end		

data/lib/attlib/search_engine_lookup.rb

@@ -0,0 +1,43 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Yali Sassoon (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+
+# This module processes the search_engines.yml file and uses it to create a global hash that 
+# is used to lookup referrers to see if they are search engines
+require 'yaml'
+
+module SearchEngineLookup
+
+	# Load search engine data stored in YAML file
+	se = YAML.load_file(File.join(File.dirname(__FILE__), '..', '..', 'data','search_engines.yml'))
+
+	# Check that none of the values for parameters in the YAML file are nil
+	se.each { | search_engine, data | if data['parameters'].nil? then puts "Problematic search engine parameter is: " + search_engine end } 
+
+	# Check that none of the values for domains in the YAML file are nil
+	se.each { | search_engine, data | if data['domains'].nil? then puts "Problematic search engine parameter is: " + search_engine end } 
+
+	# Create a hash of search engine domains, that we will perform lookups against
+	$SEARCH_ENGINE_LOOKUP = Hash.new # blank map to start with
+
+	# Now populate the lookup hash '$SEARCH_ENGINE_LOOKUP' by transforming the data from the YAML file, stored in 'se'
+	se.each do | name, data |
+		data['domains'].each do | domain |
+			new_domain = { domain => { "name" => name, "parameters" => data['parameters'] } }
+			$SEARCH_ENGINE_LOOKUP.merge!(new_domain) 
+		end
+	end
+
+end

data/lib/attlib/version.rb

@@ -0,0 +1,5 @@
+module SnowPlow
+	module Attlib
+  		VERSION = "0.0.1"
+  	end
+end

data/lib/attlib.rb

@@ -0,0 +1,6 @@
+require "attlib/version"
+require "attlib/referrer"
+
+module Attlib
+  # Your code goes here...
+end

data/spec/referrer_spec.rb

@@ -0,0 +1,21 @@
+require 'attlib'
+
+describe Referrer do
+	it "Should correctly parse Google.com search strings" do
+		ref = Referrer.new('http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari&tbo=d&biw=768&bih=900&source=lnms&tbm=isch&ei=t9fTT_TFEYb28gTtg9HZAw&sa=X&oi=mode_link&ct=mode&cd=2&sqi=2&ved=0CEUQ_AUoAQ')
+		ref.keywords.should eql "gateway oracle cards denise linn"
+	end
+
+	it "Should correct parse Google.co.uk search strings" do
+		ref = Referrer.new('http://www.google.co.uk/search?hl=en&client=safari&q=psychic+bazaar&oq=psychic+bazaa&aq=0&aqi=g1&aql=&gs_l=mobile-gws-serp.1.0.0.61498.64599.0.66559.12.9.1.1.2.2.2407.10525.6-2j0j1j3.6.0...0.0.DiYO_7K_ndg&mvs=0')
+		ref.keywords.should eql "psychic bazaar"
+	end
+
+	it "Should NOT identify Facebook as a search engine" do
+	ref = Referrer.new('http://www.facebook.com/l.php?u=http%3A%2F%2Fpsy.bz%2FLtPadV&h=MAQHYFyRRAQFzmokHhn3w4LGWVzjs7YwZGejw7Up5TqNHIw')
+	ref.is_search_engine?.should eql nil
+	end
+
+	# TO DO build out more tests, including referrers that are NOT search engines
+
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: attlib
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Yali Sassoon
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-10-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 6
+        version: "2.6"
+  type: :development
+  version_requirements: *id001
+description: Library for extracting search marketing attribution data from referrer URLs
+email: 
+- yali.sassoon@keplarllp.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE-2.0.txt
+- README.md
+- Rakefile
+- attlib.gemspec
+- data/search_engines.yml
+- lib/attlib.rb
+- lib/attlib/referrer.rb
+- lib/attlib/search_engine_lookup.rb
+- lib/attlib/version.rb
+- spec/referrer_spec.rb
+homepage: http://github.com/snowplow/attlib
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Library for extracting search marketing attribution data from referrer URLs. This is used by SnowPlow (http://github.com/snowplow/snowplow). However, our hope is that this library (and the search engines YAML) will be extended by anyone interested in parsing search engine referrer data.
+test_files: 
+- spec/referrer_spec.rb