search_engine_referer 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE +0 -0
- data/README.md +67 -0
- data/Rakefile +6 -0
- data/lib/search_engine_referer.rb +25 -0
- data/lib/search_engine_referer/base.rb +71 -0
- data/lib/search_engine_referer/bing.rb +16 -0
- data/lib/search_engine_referer/controller_helper.rb +29 -0
- data/lib/search_engine_referer/google.rb +21 -0
- data/lib/search_engine_referer/mailru.rb +16 -0
- data/lib/search_engine_referer/middleware.rb +26 -0
- data/lib/search_engine_referer/railtie.rb +11 -0
- data/lib/search_engine_referer/rambler.rb +15 -0
- data/lib/search_engine_referer/version.rb +3 -0
- data/lib/search_engine_referer/yandex.rb +21 -0
- data/search_engine_referer.gemspec +28 -0
- data/spec/search_engine_referer/base_spec.rb +49 -0
- data/spec/search_engine_referer/bing_spec.rb +33 -0
- data/spec/search_engine_referer/controller_helper_spec.rb +52 -0
- data/spec/search_engine_referer/google_spec.rb +36 -0
- data/spec/search_engine_referer/mailru_spec.rb +33 -0
- data/spec/search_engine_referer/middleware_spec.rb +65 -0
- data/spec/search_engine_referer/rambler_spec.rb +26 -0
- data/spec/search_engine_referer/yandex_spec.rb +32 -0
- data/spec/search_engine_referer_spec.rb +59 -0
- data/spec/spec_helper.rb +1 -0
- metadata +140 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e60b51384c4d2afa18171b30b7fab48cf9fea9ee
|
4
|
+
data.tar.gz: d2cb64e1d3a98a2d2360cbf17de4c39c03b18e58
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a46ac067dcd861a45f41a518e2e65136cce4e74712a5979b30a3fe53cc157c584d7ff23bc363e1105c2136fe41d0668a062432220f5425142f20c987c81b888e
|
7
|
+
data.tar.gz: b1e2133eba7c92da6071b32dcf4fdab20ba1373db8c05409dc8484bbf4da8ded16c3fbcff2d42b22b67001ba7b1e62262c425b5de02bb1a314b63f6e0c6fd46c
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
search_engine_referer
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.0.0
|
data/Gemfile
ADDED
data/LICENSE
ADDED
File without changes
|
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
#SearchEngineReferer
|
2
|
+
Get meta information from search engine`s referer for
|
3
|
+
|
4
|
+
* google.com
|
5
|
+
* bing.com
|
6
|
+
* yandex.ru
|
7
|
+
* go.mail.ru
|
8
|
+
* nova.rambler.ru
|
9
|
+
|
10
|
+
##Instalation
|
11
|
+
Add the folowing line to Gemfile and then run `bundle` command.
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'search_engine_referer'
|
15
|
+
```
|
16
|
+
|
17
|
+
##Usage
|
18
|
+
In your code use:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
referer = SearchEngineReferer.parse('http://google.com?q=ruby&start=20')
|
22
|
+
referef.search_engine # => :google
|
23
|
+
referer.page # => 3
|
24
|
+
referer.query # => 'ruby'
|
25
|
+
|
26
|
+
non_search_enfine_referer = SearchEngineReferer.parse('http://example.com?q=ruby')
|
27
|
+
non_search_enfine_referef # => nil
|
28
|
+
```
|
29
|
+
|
30
|
+
##Middleware
|
31
|
+
Tiny middleware provided. It adds `search_engine_referer` key to env with found referer. Examples:
|
32
|
+
|
33
|
+
* Uses `HTTP_REFERER` from env
|
34
|
+
|
35
|
+
```ruby
|
36
|
+
use SearchEngineReferer::Middleware
|
37
|
+
```
|
38
|
+
|
39
|
+
* Uses `CUSTOM_ENV_KEY` for referer lookup
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
use SearchEngineReferer::Middleware, 'CUSTOM_ENV_KEY'
|
43
|
+
```
|
44
|
+
|
45
|
+
* Uses provided proc for referer lookup
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
use SearchEngineReferer::Middleware do |env|
|
49
|
+
env['action_pack.cookies']['referer']
|
50
|
+
end
|
51
|
+
```
|
52
|
+
|
53
|
+
##Controller Helpers
|
54
|
+
Its add helpers for controllers and views: `search_engine_referer` and `search_engine_query`.
|
55
|
+
`search_engine_referer_source` method for referer source manipulations.
|
56
|
+
By default it returns `request.referer`. It can be overwritten in controller, for example:
|
57
|
+
|
58
|
+
```ruby
|
59
|
+
def search_engine_referer_source
|
60
|
+
cookies[:search_engine_referer] || super
|
61
|
+
end
|
62
|
+
```
|
63
|
+
|
64
|
+
**Note** that helpers will use `env['search_engine_referer']` if it provided by middleware
|
65
|
+
|
66
|
+
##Development
|
67
|
+
Fully tested pull requests are welcome.
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'search_engine_referer/version'
|
2
|
+
require 'search_engine_referer/base'
|
3
|
+
require 'search_engine_referer/google'
|
4
|
+
require 'search_engine_referer/bing'
|
5
|
+
require 'search_engine_referer/yandex'
|
6
|
+
require 'search_engine_referer/mailru'
|
7
|
+
require 'search_engine_referer/rambler'
|
8
|
+
require 'search_engine_referer/railtie' if defined? Rails
|
9
|
+
require 'search_engine_referer/middleware' if defined? Rack
|
10
|
+
|
11
|
+
module SearchEngineReferer
|
12
|
+
class << self
|
13
|
+
def parse referer_string
|
14
|
+
Base.factory(referer_string)
|
15
|
+
end
|
16
|
+
|
17
|
+
def engines
|
18
|
+
[Google, Bing, Yandex, Mailru, Rambler]
|
19
|
+
end
|
20
|
+
|
21
|
+
def engine_names
|
22
|
+
engines.map(&:name)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'rack/utils'
|
3
|
+
|
4
|
+
module SearchEngineReferer
|
5
|
+
class Base
|
6
|
+
def self.factory referer_string
|
7
|
+
url = URI.parse(referer_string)
|
8
|
+
SearchEngineReferer.engines.detect{|engine| engine.host_pattern =~ url.host}.try(:new, url)
|
9
|
+
rescue URI::InvalidURIError
|
10
|
+
nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.host_pattern
|
14
|
+
self::HOST_PATTERN
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.name
|
18
|
+
super.split('::').last.downcase.to_sym
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize url_or_referer
|
22
|
+
@url = url_or_referer if url_or_referer.is_a?(URI)
|
23
|
+
@source = url_or_referer.to_s
|
24
|
+
end
|
25
|
+
|
26
|
+
attr_reader :source
|
27
|
+
|
28
|
+
def query
|
29
|
+
query!.strip
|
30
|
+
end
|
31
|
+
|
32
|
+
def search_engine
|
33
|
+
self.class.to_s.split('::').last.downcase.to_sym
|
34
|
+
end
|
35
|
+
|
36
|
+
def page
|
37
|
+
@page ||= page!
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_hash
|
41
|
+
{
|
42
|
+
:search_engine => search_engine,
|
43
|
+
:source => source,
|
44
|
+
:query => query,
|
45
|
+
:page => page
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
def == other
|
50
|
+
source == other.source
|
51
|
+
end
|
52
|
+
|
53
|
+
protected
|
54
|
+
|
55
|
+
def url
|
56
|
+
@url ||= URI.parse(source)
|
57
|
+
end
|
58
|
+
|
59
|
+
def user_query_key
|
60
|
+
raise NotImplementedError
|
61
|
+
end
|
62
|
+
|
63
|
+
def params
|
64
|
+
@params ||= Rack::Utils.parse_query(url.query) rescue {}
|
65
|
+
end
|
66
|
+
|
67
|
+
def query!
|
68
|
+
params[user_query_key] || ''
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
require 'active_support/core_ext/object/try'
|
3
|
+
|
4
|
+
module SearchEngineReferer
|
5
|
+
module ControllerHelper
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
helper_method :search_engine_referer, :search_engine_query
|
10
|
+
end
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
def search_engine_referer
|
15
|
+
@search_engine_referer ||= (
|
16
|
+
env['search_engine_referer'] ||
|
17
|
+
SearchEngineReferer.parse(search_engine_referer_source)
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def search_engine_query
|
22
|
+
search_engine_referer.try(:query) || ''
|
23
|
+
end
|
24
|
+
|
25
|
+
def search_engine_referer_source
|
26
|
+
request.referer
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Google < Base
|
3
|
+
DEFAULT_RESULTS_PER_PAGE = 10
|
4
|
+
HOST_PATTERN = /^(www\.)?google\./
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def page!
|
9
|
+
params['start'].to_i / results_per_page + 1
|
10
|
+
end
|
11
|
+
|
12
|
+
def user_query_key
|
13
|
+
'q'
|
14
|
+
end
|
15
|
+
|
16
|
+
def results_per_page
|
17
|
+
result = params['num'].to_i
|
18
|
+
result.zero? ? DEFAULT_RESULTS_PER_PAGE : result
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Mailru < Base
|
3
|
+
DEFAULT_RESULTS_PER_PAGE = 10
|
4
|
+
HOST_PATTERN = /go\.mail\.ru/
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def page!
|
9
|
+
params['sf'].to_i / DEFAULT_RESULTS_PER_PAGE + 1
|
10
|
+
end
|
11
|
+
|
12
|
+
def user_query_key
|
13
|
+
'q'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Middleware
|
3
|
+
def initialize app, referer_source = nil, &block
|
4
|
+
@app = app
|
5
|
+
@referer_source = referer_source || block
|
6
|
+
end
|
7
|
+
|
8
|
+
def call env
|
9
|
+
env['search_engine_referer'] = SearchEngineReferer.parse(referer(env))
|
10
|
+
@app.call(env)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def referer env
|
16
|
+
case
|
17
|
+
when env.key?(@referer_source)
|
18
|
+
env[@referer_source]
|
19
|
+
when @referer_source.respond_to?(:call)
|
20
|
+
@referer_source.call(env)
|
21
|
+
else
|
22
|
+
env['HTTP_REFERER']
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'search_engine_referer/controller_helper'
|
2
|
+
|
3
|
+
module SearchEngineReferer
|
4
|
+
class Railtie < Rails::Railtie
|
5
|
+
initializer 'search_engine_referer.controller_helper' do
|
6
|
+
ActiveSupport.on_load :action_controller do
|
7
|
+
include ControllerHelper
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Yandex < Base
|
3
|
+
HOST_PATTERN = /^(www\.)?yandex\./
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def page!
|
8
|
+
params['p'].to_i + 1
|
9
|
+
end
|
10
|
+
|
11
|
+
def user_query_key
|
12
|
+
'text'
|
13
|
+
end
|
14
|
+
|
15
|
+
def query!
|
16
|
+
result = super
|
17
|
+
result = result.encode(Encoding::UTF_8, Encoding::CP1251) unless result.valid_encoding?
|
18
|
+
result
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
require 'search_engine_referer/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'search_engine_referer'
|
7
|
+
s.version = SearchEngineReferer::VERSION
|
8
|
+
s.authors = ['Alexander Stanko']
|
9
|
+
s.email = ['astanko@aviasales.ru']
|
10
|
+
s.homepage = 'http://github.com/Rakoth/search_engine_referer'
|
11
|
+
s.summary = %q{Parse search engine referer}
|
12
|
+
s.description = %q{
|
13
|
+
Help to parse referer url for get user search query and page of search engine results.
|
14
|
+
Work with Google and Yandex for now.
|
15
|
+
}
|
16
|
+
|
17
|
+
s.rubyforge_project = 'search_engine_referer'
|
18
|
+
|
19
|
+
s.files = `git ls-files`.split("\n")
|
20
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
21
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
22
|
+
s.require_paths = ['lib']
|
23
|
+
|
24
|
+
s.add_development_dependency 'rspec'
|
25
|
+
s.add_development_dependency 'activesupport'
|
26
|
+
s.add_development_dependency 'rack-test'
|
27
|
+
s.add_runtime_dependency 'rack'
|
28
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Base do
|
4
|
+
describe '.factory' do
|
5
|
+
it 'should build correct parser with factory' do
|
6
|
+
SearchEngineReferer::Base.factory('http://google.com').
|
7
|
+
should be_an_instance_of(SearchEngineReferer::Google)
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'should build correct parser for yandex referer' do
|
11
|
+
SearchEngineReferer::Base.factory('http://yandex.ru').
|
12
|
+
should be_an_instance_of(SearchEngineReferer::Yandex)
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'should build correct parser for mailru referer' do
|
16
|
+
SearchEngineReferer::Base.factory('http://go.mail.ru').
|
17
|
+
should be_an_instance_of(SearchEngineReferer::Mailru)
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should build correct parser for rambler referer' do
|
21
|
+
SearchEngineReferer::Base.factory('http://nova.rambler.ru').
|
22
|
+
should be_an_instance_of(SearchEngineReferer::Rambler)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should build correct parser for rambler referer' do
|
26
|
+
SearchEngineReferer::Base.factory('http://bing.com').
|
27
|
+
should be_an_instance_of(SearchEngineReferer::Bing)
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'should return nil if referer cannot be parsed' do
|
31
|
+
SearchEngineReferer::Base.factory('http://test.ru').should be_nil
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should return nil if referer cannot be parsed (goggle in host name)' do
|
35
|
+
SearchEngineReferer::Base.factory('http://avia.newgoogle.ru').should be_nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe '#to_hash' do
|
40
|
+
it 'should provide all parsed params as hash' do
|
41
|
+
SearchEngineReferer::Yandex.new('http://yandex.ru?text=test&p=1').to_hash.should == {
|
42
|
+
:source => 'http://yandex.ru?text=test&p=1',
|
43
|
+
:page => 2,
|
44
|
+
:query => 'test',
|
45
|
+
:search_engine => :yandex
|
46
|
+
}
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Bing do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be bing' do
|
7
|
+
parser.new('http://bing.com').search_engine.should == :bing
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://bing.com?first=1').page.should == 1
|
14
|
+
parser.new('http://bing.com?first=11').page.should == 2
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should parse page if it not specified in query' do
|
18
|
+
parser.new('http://bing.com').page.should == 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#query' do
|
23
|
+
it 'should parse user query correctly from text parameter' do
|
24
|
+
parser.new(URI.escape('http://bing.com?q=test+user+request')).query.
|
25
|
+
should == 'test user request'
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should parse user query with unicode symbols' do
|
29
|
+
parser.new(URI.escape('http://bing.com?q=тестовый+запрос')).query.
|
30
|
+
should == 'тестовый запрос'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'search_engine_referer/controller_helper'
|
3
|
+
|
4
|
+
class Controller
|
5
|
+
def initialize request
|
6
|
+
@request = request
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.helpers
|
10
|
+
@helpers ||= []
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.helper_method *args
|
14
|
+
helpers.push(*args)
|
15
|
+
end
|
16
|
+
|
17
|
+
def env
|
18
|
+
@env ||= {}
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_accessor :request
|
22
|
+
|
23
|
+
include SearchEngineReferer::ControllerHelper
|
24
|
+
end
|
25
|
+
|
26
|
+
describe SearchEngineReferer::ControllerHelper do
|
27
|
+
it 'should add helpers to controller when included' do
|
28
|
+
Controller.helpers.should =~ [:search_engine_query, :search_engine_referer]
|
29
|
+
end
|
30
|
+
|
31
|
+
describe 'search engine helpers' do
|
32
|
+
subject{Controller.new(request)}
|
33
|
+
let(:referer_source){'http://yandex.ru?text=test'}
|
34
|
+
let(:referer){SearchEngineReferer::Yandex.new(referer_source)}
|
35
|
+
|
36
|
+
context 'referer given in headers' do
|
37
|
+
let(:request){double(referer: referer_source)}
|
38
|
+
its(:search_engine_query){should == 'test'}
|
39
|
+
its(:search_engine_referer){should == referer}
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'with search engine referer middleware' do
|
43
|
+
before do
|
44
|
+
subject.env['search_engine_referer'] = referer
|
45
|
+
end
|
46
|
+
|
47
|
+
let(:request){double(referer: nil)}
|
48
|
+
its(:search_engine_query){should == 'test'}
|
49
|
+
its(:search_engine_referer){should == referer}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Google do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be google' do
|
7
|
+
parser.new('http://google.com').search_engine.should == :google
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://google.ru?start=10').page.should == 2
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page with custom results on page user setting' do
|
17
|
+
parser.new('http://google.com?start=200&num=100').page.should == 3
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should parse page if it not specified in query' do
|
21
|
+
parser.new('http://google.ru').page.should == 1
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#query' do
|
26
|
+
it 'should parse user query correctly from text parameter' do
|
27
|
+
parser.new(URI.escape('http://google.ru?q=test user request')).query.
|
28
|
+
should == 'test user request'
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should parse user query with unicode symbols' do
|
32
|
+
parser.new(URI.escape('http://google.ru?q=тестовый запрос')).query.
|
33
|
+
should == 'тестовый запрос'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Mailru do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be mailru' do
|
7
|
+
parser.new('http://go.mail.ru').search_engine.should == :mailru
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://go.mail.ru?rch=l&sf=0').page.should == 1
|
14
|
+
parser.new('http://go.mail.ru?rch=l&sf=10').page.should == 2
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should parse page if it not specified in query' do
|
18
|
+
parser.new('http://go.mail.ru').page.should == 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#query' do
|
23
|
+
it 'should parse user query correctly from text parameter' do
|
24
|
+
parser.new(URI.escape('http://go.mail.ru?q=test+user+request')).query.
|
25
|
+
should == 'test user request'
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should parse user query with unicode symbols' do
|
29
|
+
parser.new(URI.escape('http://go.mail.ru?q=тестовый+запрос')).query.
|
30
|
+
should == 'тестовый запрос'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'search_engine_referer/middleware'
|
3
|
+
require 'rack/test'
|
4
|
+
|
5
|
+
describe SearchEngineReferer::Middleware do
|
6
|
+
include Rack::Test::Methods
|
7
|
+
|
8
|
+
describe 'referer source initialization' do
|
9
|
+
context 'env key' do
|
10
|
+
let(:app) do
|
11
|
+
Rack::Builder.new do
|
12
|
+
use SearchEngineReferer::Middleware, 'HTTP_CUSTOM_REFERER'
|
13
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'uses custom env key' do
|
18
|
+
get '/', {}, 'HTTP_CUSTOM_REFERER' => 'http://google.com?q=test'
|
19
|
+
last_response[:referer].should be
|
20
|
+
last_response[:referer].search_engine.should == :google
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'env proc' do
|
25
|
+
let(:app) do
|
26
|
+
Rack::Builder.new do
|
27
|
+
use SearchEngineReferer::Middleware do |env|
|
28
|
+
env['HTTP_CUSTOM_REFERER'] || env['HTTP_REFERER']
|
29
|
+
end
|
30
|
+
|
31
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'uses proc to evaluate referer from env' do
|
36
|
+
get '/', {},
|
37
|
+
'HTTP_CUSTOM_REFERER' => 'http://google.com?q=test',
|
38
|
+
'HTTP_REFERER' => 'http://bing.com?q=test1'
|
39
|
+
last_response[:referer].should be
|
40
|
+
last_response[:referer].search_engine.should == :google
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'uses proc to evaluate referer from env on every request' do
|
44
|
+
get '/', {}, 'HTTP_REFERER' => 'http://bing.com?q=test1'
|
45
|
+
last_response[:referer].should be
|
46
|
+
last_response[:referer].search_engine.should == :bing
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'default' do
|
51
|
+
let(:app) do
|
52
|
+
Rack::Builder.new do
|
53
|
+
use SearchEngineReferer::Middleware
|
54
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'uses HTTP_REFERER by ' do
|
59
|
+
get '/', {}, 'HTTP_REFERER' => 'http://google.com?q=test'
|
60
|
+
last_response[:referer].should be
|
61
|
+
last_response[:referer].search_engine.should == :google
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Rambler do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be rambler' do
|
7
|
+
parser.new('http://nova.rambler.ru').search_engine.should == :rambler
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it is not specified' do
|
13
|
+
parser.new('http://nova.rambler.ru?query=test').page.should == 1
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page if it is specified in url' do
|
17
|
+
parser.new('http://nova.rambler.ru?query=test&page=2').page.should == 2
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#query' do
|
22
|
+
it 'should parse page if it is not specified' do
|
23
|
+
parser.new('http://nova.rambler.ru?query=test').query.should == 'test'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Yandex do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be yandex' do
|
7
|
+
parser.new('http://yandex.ru').search_engine.should == :yandex
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://yandex.ru?p=2').page.should == 3
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page if it not specified in query' do
|
17
|
+
parser.new('http://yandex.ru').page.should == 1
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#query' do
|
22
|
+
it 'should parse user query correctly from text parameter' do
|
23
|
+
parser.new(URI.escape('http://yandex.ru?text=test user request')).
|
24
|
+
query.should == 'test user request'
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should parse user query with unicode symbols' do
|
28
|
+
parser.new(URI.escape('http://yandex.ru?text=тестовый запрос')).
|
29
|
+
query.should == 'тестовый запрос'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer do
|
4
|
+
describe '.engines' do
|
5
|
+
it 'should return all supported engines' do
|
6
|
+
SearchEngineReferer.engine_names.should =~ [:yandex, :mailru, :google, :rambler, :bing]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.parse' do
|
11
|
+
it 'should return referer or nil' do
|
12
|
+
SearchEngineReferer.parse('http://yandex.ru').
|
13
|
+
should be_kind_of(SearchEngineReferer::Base)
|
14
|
+
|
15
|
+
SearchEngineReferer.parse('http://test.ru').should be_nil
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should handle empty referer' do
|
19
|
+
SearchEngineReferer.parse(nil).should be_nil
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should handle known domains only' do
|
23
|
+
SearchEngineReferer.parse('http://test.ru?q=1&text=2').should be_nil
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'should handle google referer' do
|
27
|
+
SearchEngineReferer.parse('http://google.ru?q=1&text=2').query.should == '1'
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'should handle yandex referer' do
|
31
|
+
SearchEngineReferer.parse('http://yandex.ru?q=1&text=2').query.should == '2'
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should handle rambler referer' do
|
35
|
+
SearchEngineReferer.parse('http://nova.rambler.ru?query=1&page=2').query.should == '1'
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should unescape params found' do
|
39
|
+
SearchEngineReferer.parse('http://google.ru?q=1+2').query.should == '1 2'
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should handle urls without query params' do
|
43
|
+
SearchEngineReferer.parse('http://google.ru').query.should == ''
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should return nil on incorrect referer' do
|
47
|
+
SearchEngineReferer.parse('http://www.aviasales.ru/\\').should be_nil
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'should return nil on incorrect referer' do
|
51
|
+
SearchEngineReferer.parse('string with spaces').should be_nil
|
52
|
+
end
|
53
|
+
|
54
|
+
it 'should use iconv for cp1251 strings from yandex' do
|
55
|
+
SearchEngineReferer.parse('http://yandex.ru/yandsearch?text=%e0%e2%e8%e0%f1%e5%eb%f1').
|
56
|
+
query.should == 'авиаселс'
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'search_engine_referer'
|
metadata
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: search_engine_referer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alexander Stanko
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-12-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rspec
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activesupport
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rack-test
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rack
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: "\n Help to parse referer url for get user search query and page of
|
70
|
+
search engine results.\n Work with Google and Yandex for now.\n "
|
71
|
+
email:
|
72
|
+
- astanko@aviasales.ru
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- .gitignore
|
78
|
+
- .rspec
|
79
|
+
- .ruby-gemset
|
80
|
+
- .ruby-version
|
81
|
+
- Gemfile
|
82
|
+
- LICENSE
|
83
|
+
- README.md
|
84
|
+
- Rakefile
|
85
|
+
- lib/search_engine_referer.rb
|
86
|
+
- lib/search_engine_referer/base.rb
|
87
|
+
- lib/search_engine_referer/bing.rb
|
88
|
+
- lib/search_engine_referer/controller_helper.rb
|
89
|
+
- lib/search_engine_referer/google.rb
|
90
|
+
- lib/search_engine_referer/mailru.rb
|
91
|
+
- lib/search_engine_referer/middleware.rb
|
92
|
+
- lib/search_engine_referer/railtie.rb
|
93
|
+
- lib/search_engine_referer/rambler.rb
|
94
|
+
- lib/search_engine_referer/version.rb
|
95
|
+
- lib/search_engine_referer/yandex.rb
|
96
|
+
- search_engine_referer.gemspec
|
97
|
+
- spec/search_engine_referer/base_spec.rb
|
98
|
+
- spec/search_engine_referer/bing_spec.rb
|
99
|
+
- spec/search_engine_referer/controller_helper_spec.rb
|
100
|
+
- spec/search_engine_referer/google_spec.rb
|
101
|
+
- spec/search_engine_referer/mailru_spec.rb
|
102
|
+
- spec/search_engine_referer/middleware_spec.rb
|
103
|
+
- spec/search_engine_referer/rambler_spec.rb
|
104
|
+
- spec/search_engine_referer/yandex_spec.rb
|
105
|
+
- spec/search_engine_referer_spec.rb
|
106
|
+
- spec/spec_helper.rb
|
107
|
+
homepage: http://github.com/Rakoth/search_engine_referer
|
108
|
+
licenses: []
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project: search_engine_referer
|
126
|
+
rubygems_version: 2.0.3
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Parse search engine referer
|
130
|
+
test_files:
|
131
|
+
- spec/search_engine_referer/base_spec.rb
|
132
|
+
- spec/search_engine_referer/bing_spec.rb
|
133
|
+
- spec/search_engine_referer/controller_helper_spec.rb
|
134
|
+
- spec/search_engine_referer/google_spec.rb
|
135
|
+
- spec/search_engine_referer/mailru_spec.rb
|
136
|
+
- spec/search_engine_referer/middleware_spec.rb
|
137
|
+
- spec/search_engine_referer/rambler_spec.rb
|
138
|
+
- spec/search_engine_referer/yandex_spec.rb
|
139
|
+
- spec/search_engine_referer_spec.rb
|
140
|
+
- spec/spec_helper.rb
|