search_engine_referer 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE +0 -0
- data/README.md +67 -0
- data/Rakefile +6 -0
- data/lib/search_engine_referer.rb +25 -0
- data/lib/search_engine_referer/base.rb +71 -0
- data/lib/search_engine_referer/bing.rb +16 -0
- data/lib/search_engine_referer/controller_helper.rb +29 -0
- data/lib/search_engine_referer/google.rb +21 -0
- data/lib/search_engine_referer/mailru.rb +16 -0
- data/lib/search_engine_referer/middleware.rb +26 -0
- data/lib/search_engine_referer/railtie.rb +11 -0
- data/lib/search_engine_referer/rambler.rb +15 -0
- data/lib/search_engine_referer/version.rb +3 -0
- data/lib/search_engine_referer/yandex.rb +21 -0
- data/search_engine_referer.gemspec +28 -0
- data/spec/search_engine_referer/base_spec.rb +49 -0
- data/spec/search_engine_referer/bing_spec.rb +33 -0
- data/spec/search_engine_referer/controller_helper_spec.rb +52 -0
- data/spec/search_engine_referer/google_spec.rb +36 -0
- data/spec/search_engine_referer/mailru_spec.rb +33 -0
- data/spec/search_engine_referer/middleware_spec.rb +65 -0
- data/spec/search_engine_referer/rambler_spec.rb +26 -0
- data/spec/search_engine_referer/yandex_spec.rb +32 -0
- data/spec/search_engine_referer_spec.rb +59 -0
- data/spec/spec_helper.rb +1 -0
- metadata +140 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e60b51384c4d2afa18171b30b7fab48cf9fea9ee
|
4
|
+
data.tar.gz: d2cb64e1d3a98a2d2360cbf17de4c39c03b18e58
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a46ac067dcd861a45f41a518e2e65136cce4e74712a5979b30a3fe53cc157c584d7ff23bc363e1105c2136fe41d0668a062432220f5425142f20c987c81b888e
|
7
|
+
data.tar.gz: b1e2133eba7c92da6071b32dcf4fdab20ba1373db8c05409dc8484bbf4da8ded16c3fbcff2d42b22b67001ba7b1e62262c425b5de02bb1a314b63f6e0c6fd46c
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
search_engine_referer
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.0.0
|
data/Gemfile
ADDED
data/LICENSE
ADDED
File without changes
|
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
#SearchEngineReferer
|
2
|
+
Get meta information from search engine`s referer for
|
3
|
+
|
4
|
+
* google.com
|
5
|
+
* bing.com
|
6
|
+
* yandex.ru
|
7
|
+
* go.mail.ru
|
8
|
+
* nova.rambler.ru
|
9
|
+
|
10
|
+
##Instalation
|
11
|
+
Add the folowing line to Gemfile and then run `bundle` command.
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'search_engine_referer'
|
15
|
+
```
|
16
|
+
|
17
|
+
##Usage
|
18
|
+
In your code use:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
referer = SearchEngineReferer.parse('http://google.com?q=ruby&start=20')
|
22
|
+
referef.search_engine # => :google
|
23
|
+
referer.page # => 3
|
24
|
+
referer.query # => 'ruby'
|
25
|
+
|
26
|
+
non_search_enfine_referer = SearchEngineReferer.parse('http://example.com?q=ruby')
|
27
|
+
non_search_enfine_referef # => nil
|
28
|
+
```
|
29
|
+
|
30
|
+
##Middleware
|
31
|
+
Tiny middleware provided. It adds `search_engine_referer` key to env with found referer. Examples:
|
32
|
+
|
33
|
+
* Uses `HTTP_REFERER` from env
|
34
|
+
|
35
|
+
```ruby
|
36
|
+
use SearchEngineReferer::Middleware
|
37
|
+
```
|
38
|
+
|
39
|
+
* Uses `CUSTOM_ENV_KEY` for referer lookup
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
use SearchEngineReferer::Middleware, 'CUSTOM_ENV_KEY'
|
43
|
+
```
|
44
|
+
|
45
|
+
* Uses provided proc for referer lookup
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
use SearchEngineReferer::Middleware do |env|
|
49
|
+
env['action_pack.cookies']['referer']
|
50
|
+
end
|
51
|
+
```
|
52
|
+
|
53
|
+
##Controller Helpers
|
54
|
+
Its add helpers for controllers and views: `search_engine_referer` and `search_engine_query`.
|
55
|
+
`search_engine_referer_source` method for referer source manipulations.
|
56
|
+
By default it returns `request.referer`. It can be overwritten in controller, for example:
|
57
|
+
|
58
|
+
```ruby
|
59
|
+
def search_engine_referer_source
|
60
|
+
cookies[:search_engine_referer] || super
|
61
|
+
end
|
62
|
+
```
|
63
|
+
|
64
|
+
**Note** that helpers will use `env['search_engine_referer']` if it provided by middleware
|
65
|
+
|
66
|
+
##Development
|
67
|
+
Fully tested pull requests are welcome.
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'search_engine_referer/version'
|
2
|
+
require 'search_engine_referer/base'
|
3
|
+
require 'search_engine_referer/google'
|
4
|
+
require 'search_engine_referer/bing'
|
5
|
+
require 'search_engine_referer/yandex'
|
6
|
+
require 'search_engine_referer/mailru'
|
7
|
+
require 'search_engine_referer/rambler'
|
8
|
+
require 'search_engine_referer/railtie' if defined? Rails
|
9
|
+
require 'search_engine_referer/middleware' if defined? Rack
|
10
|
+
|
11
|
+
module SearchEngineReferer
|
12
|
+
class << self
|
13
|
+
def parse referer_string
|
14
|
+
Base.factory(referer_string)
|
15
|
+
end
|
16
|
+
|
17
|
+
def engines
|
18
|
+
[Google, Bing, Yandex, Mailru, Rambler]
|
19
|
+
end
|
20
|
+
|
21
|
+
def engine_names
|
22
|
+
engines.map(&:name)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'rack/utils'
|
3
|
+
|
4
|
+
module SearchEngineReferer
|
5
|
+
class Base
|
6
|
+
def self.factory referer_string
|
7
|
+
url = URI.parse(referer_string)
|
8
|
+
SearchEngineReferer.engines.detect{|engine| engine.host_pattern =~ url.host}.try(:new, url)
|
9
|
+
rescue URI::InvalidURIError
|
10
|
+
nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.host_pattern
|
14
|
+
self::HOST_PATTERN
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.name
|
18
|
+
super.split('::').last.downcase.to_sym
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize url_or_referer
|
22
|
+
@url = url_or_referer if url_or_referer.is_a?(URI)
|
23
|
+
@source = url_or_referer.to_s
|
24
|
+
end
|
25
|
+
|
26
|
+
attr_reader :source
|
27
|
+
|
28
|
+
def query
|
29
|
+
query!.strip
|
30
|
+
end
|
31
|
+
|
32
|
+
def search_engine
|
33
|
+
self.class.to_s.split('::').last.downcase.to_sym
|
34
|
+
end
|
35
|
+
|
36
|
+
def page
|
37
|
+
@page ||= page!
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_hash
|
41
|
+
{
|
42
|
+
:search_engine => search_engine,
|
43
|
+
:source => source,
|
44
|
+
:query => query,
|
45
|
+
:page => page
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
def == other
|
50
|
+
source == other.source
|
51
|
+
end
|
52
|
+
|
53
|
+
protected
|
54
|
+
|
55
|
+
def url
|
56
|
+
@url ||= URI.parse(source)
|
57
|
+
end
|
58
|
+
|
59
|
+
def user_query_key
|
60
|
+
raise NotImplementedError
|
61
|
+
end
|
62
|
+
|
63
|
+
def params
|
64
|
+
@params ||= Rack::Utils.parse_query(url.query) rescue {}
|
65
|
+
end
|
66
|
+
|
67
|
+
def query!
|
68
|
+
params[user_query_key] || ''
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
require 'active_support/core_ext/object/try'
|
3
|
+
|
4
|
+
module SearchEngineReferer
|
5
|
+
module ControllerHelper
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
helper_method :search_engine_referer, :search_engine_query
|
10
|
+
end
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
def search_engine_referer
|
15
|
+
@search_engine_referer ||= (
|
16
|
+
env['search_engine_referer'] ||
|
17
|
+
SearchEngineReferer.parse(search_engine_referer_source)
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def search_engine_query
|
22
|
+
search_engine_referer.try(:query) || ''
|
23
|
+
end
|
24
|
+
|
25
|
+
def search_engine_referer_source
|
26
|
+
request.referer
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Google < Base
|
3
|
+
DEFAULT_RESULTS_PER_PAGE = 10
|
4
|
+
HOST_PATTERN = /^(www\.)?google\./
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def page!
|
9
|
+
params['start'].to_i / results_per_page + 1
|
10
|
+
end
|
11
|
+
|
12
|
+
def user_query_key
|
13
|
+
'q'
|
14
|
+
end
|
15
|
+
|
16
|
+
def results_per_page
|
17
|
+
result = params['num'].to_i
|
18
|
+
result.zero? ? DEFAULT_RESULTS_PER_PAGE : result
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Mailru < Base
|
3
|
+
DEFAULT_RESULTS_PER_PAGE = 10
|
4
|
+
HOST_PATTERN = /go\.mail\.ru/
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def page!
|
9
|
+
params['sf'].to_i / DEFAULT_RESULTS_PER_PAGE + 1
|
10
|
+
end
|
11
|
+
|
12
|
+
def user_query_key
|
13
|
+
'q'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Middleware
|
3
|
+
def initialize app, referer_source = nil, &block
|
4
|
+
@app = app
|
5
|
+
@referer_source = referer_source || block
|
6
|
+
end
|
7
|
+
|
8
|
+
def call env
|
9
|
+
env['search_engine_referer'] = SearchEngineReferer.parse(referer(env))
|
10
|
+
@app.call(env)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def referer env
|
16
|
+
case
|
17
|
+
when env.key?(@referer_source)
|
18
|
+
env[@referer_source]
|
19
|
+
when @referer_source.respond_to?(:call)
|
20
|
+
@referer_source.call(env)
|
21
|
+
else
|
22
|
+
env['HTTP_REFERER']
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'search_engine_referer/controller_helper'
|
2
|
+
|
3
|
+
module SearchEngineReferer
|
4
|
+
class Railtie < Rails::Railtie
|
5
|
+
initializer 'search_engine_referer.controller_helper' do
|
6
|
+
ActiveSupport.on_load :action_controller do
|
7
|
+
include ControllerHelper
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module SearchEngineReferer
|
2
|
+
class Yandex < Base
|
3
|
+
HOST_PATTERN = /^(www\.)?yandex\./
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def page!
|
8
|
+
params['p'].to_i + 1
|
9
|
+
end
|
10
|
+
|
11
|
+
def user_query_key
|
12
|
+
'text'
|
13
|
+
end
|
14
|
+
|
15
|
+
def query!
|
16
|
+
result = super
|
17
|
+
result = result.encode(Encoding::UTF_8, Encoding::CP1251) unless result.valid_encoding?
|
18
|
+
result
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
require 'search_engine_referer/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'search_engine_referer'
|
7
|
+
s.version = SearchEngineReferer::VERSION
|
8
|
+
s.authors = ['Alexander Stanko']
|
9
|
+
s.email = ['astanko@aviasales.ru']
|
10
|
+
s.homepage = 'http://github.com/Rakoth/search_engine_referer'
|
11
|
+
s.summary = %q{Parse search engine referer}
|
12
|
+
s.description = %q{
|
13
|
+
Help to parse referer url for get user search query and page of search engine results.
|
14
|
+
Work with Google and Yandex for now.
|
15
|
+
}
|
16
|
+
|
17
|
+
s.rubyforge_project = 'search_engine_referer'
|
18
|
+
|
19
|
+
s.files = `git ls-files`.split("\n")
|
20
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
21
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
22
|
+
s.require_paths = ['lib']
|
23
|
+
|
24
|
+
s.add_development_dependency 'rspec'
|
25
|
+
s.add_development_dependency 'activesupport'
|
26
|
+
s.add_development_dependency 'rack-test'
|
27
|
+
s.add_runtime_dependency 'rack'
|
28
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Base do
|
4
|
+
describe '.factory' do
|
5
|
+
it 'should build correct parser with factory' do
|
6
|
+
SearchEngineReferer::Base.factory('http://google.com').
|
7
|
+
should be_an_instance_of(SearchEngineReferer::Google)
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'should build correct parser for yandex referer' do
|
11
|
+
SearchEngineReferer::Base.factory('http://yandex.ru').
|
12
|
+
should be_an_instance_of(SearchEngineReferer::Yandex)
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'should build correct parser for mailru referer' do
|
16
|
+
SearchEngineReferer::Base.factory('http://go.mail.ru').
|
17
|
+
should be_an_instance_of(SearchEngineReferer::Mailru)
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should build correct parser for rambler referer' do
|
21
|
+
SearchEngineReferer::Base.factory('http://nova.rambler.ru').
|
22
|
+
should be_an_instance_of(SearchEngineReferer::Rambler)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should build correct parser for rambler referer' do
|
26
|
+
SearchEngineReferer::Base.factory('http://bing.com').
|
27
|
+
should be_an_instance_of(SearchEngineReferer::Bing)
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'should return nil if referer cannot be parsed' do
|
31
|
+
SearchEngineReferer::Base.factory('http://test.ru').should be_nil
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should return nil if referer cannot be parsed (goggle in host name)' do
|
35
|
+
SearchEngineReferer::Base.factory('http://avia.newgoogle.ru').should be_nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe '#to_hash' do
|
40
|
+
it 'should provide all parsed params as hash' do
|
41
|
+
SearchEngineReferer::Yandex.new('http://yandex.ru?text=test&p=1').to_hash.should == {
|
42
|
+
:source => 'http://yandex.ru?text=test&p=1',
|
43
|
+
:page => 2,
|
44
|
+
:query => 'test',
|
45
|
+
:search_engine => :yandex
|
46
|
+
}
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Bing do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be bing' do
|
7
|
+
parser.new('http://bing.com').search_engine.should == :bing
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://bing.com?first=1').page.should == 1
|
14
|
+
parser.new('http://bing.com?first=11').page.should == 2
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should parse page if it not specified in query' do
|
18
|
+
parser.new('http://bing.com').page.should == 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#query' do
|
23
|
+
it 'should parse user query correctly from text parameter' do
|
24
|
+
parser.new(URI.escape('http://bing.com?q=test+user+request')).query.
|
25
|
+
should == 'test user request'
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should parse user query with unicode symbols' do
|
29
|
+
parser.new(URI.escape('http://bing.com?q=тестовый+запрос')).query.
|
30
|
+
should == 'тестовый запрос'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'search_engine_referer/controller_helper'
|
3
|
+
|
4
|
+
class Controller
|
5
|
+
def initialize request
|
6
|
+
@request = request
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.helpers
|
10
|
+
@helpers ||= []
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.helper_method *args
|
14
|
+
helpers.push(*args)
|
15
|
+
end
|
16
|
+
|
17
|
+
def env
|
18
|
+
@env ||= {}
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_accessor :request
|
22
|
+
|
23
|
+
include SearchEngineReferer::ControllerHelper
|
24
|
+
end
|
25
|
+
|
26
|
+
describe SearchEngineReferer::ControllerHelper do
|
27
|
+
it 'should add helpers to controller when included' do
|
28
|
+
Controller.helpers.should =~ [:search_engine_query, :search_engine_referer]
|
29
|
+
end
|
30
|
+
|
31
|
+
describe 'search engine helpers' do
|
32
|
+
subject{Controller.new(request)}
|
33
|
+
let(:referer_source){'http://yandex.ru?text=test'}
|
34
|
+
let(:referer){SearchEngineReferer::Yandex.new(referer_source)}
|
35
|
+
|
36
|
+
context 'referer given in headers' do
|
37
|
+
let(:request){double(referer: referer_source)}
|
38
|
+
its(:search_engine_query){should == 'test'}
|
39
|
+
its(:search_engine_referer){should == referer}
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'with search engine referer middleware' do
|
43
|
+
before do
|
44
|
+
subject.env['search_engine_referer'] = referer
|
45
|
+
end
|
46
|
+
|
47
|
+
let(:request){double(referer: nil)}
|
48
|
+
its(:search_engine_query){should == 'test'}
|
49
|
+
its(:search_engine_referer){should == referer}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Google do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be google' do
|
7
|
+
parser.new('http://google.com').search_engine.should == :google
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://google.ru?start=10').page.should == 2
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page with custom results on page user setting' do
|
17
|
+
parser.new('http://google.com?start=200&num=100').page.should == 3
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should parse page if it not specified in query' do
|
21
|
+
parser.new('http://google.ru').page.should == 1
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#query' do
|
26
|
+
it 'should parse user query correctly from text parameter' do
|
27
|
+
parser.new(URI.escape('http://google.ru?q=test user request')).query.
|
28
|
+
should == 'test user request'
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should parse user query with unicode symbols' do
|
32
|
+
parser.new(URI.escape('http://google.ru?q=тестовый запрос')).query.
|
33
|
+
should == 'тестовый запрос'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Mailru do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be mailru' do
|
7
|
+
parser.new('http://go.mail.ru').search_engine.should == :mailru
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://go.mail.ru?rch=l&sf=0').page.should == 1
|
14
|
+
parser.new('http://go.mail.ru?rch=l&sf=10').page.should == 2
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should parse page if it not specified in query' do
|
18
|
+
parser.new('http://go.mail.ru').page.should == 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#query' do
|
23
|
+
it 'should parse user query correctly from text parameter' do
|
24
|
+
parser.new(URI.escape('http://go.mail.ru?q=test+user+request')).query.
|
25
|
+
should == 'test user request'
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should parse user query with unicode symbols' do
|
29
|
+
parser.new(URI.escape('http://go.mail.ru?q=тестовый+запрос')).query.
|
30
|
+
should == 'тестовый запрос'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'search_engine_referer/middleware'
|
3
|
+
require 'rack/test'
|
4
|
+
|
5
|
+
describe SearchEngineReferer::Middleware do
|
6
|
+
include Rack::Test::Methods
|
7
|
+
|
8
|
+
describe 'referer source initialization' do
|
9
|
+
context 'env key' do
|
10
|
+
let(:app) do
|
11
|
+
Rack::Builder.new do
|
12
|
+
use SearchEngineReferer::Middleware, 'HTTP_CUSTOM_REFERER'
|
13
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'uses custom env key' do
|
18
|
+
get '/', {}, 'HTTP_CUSTOM_REFERER' => 'http://google.com?q=test'
|
19
|
+
last_response[:referer].should be
|
20
|
+
last_response[:referer].search_engine.should == :google
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'env proc' do
|
25
|
+
let(:app) do
|
26
|
+
Rack::Builder.new do
|
27
|
+
use SearchEngineReferer::Middleware do |env|
|
28
|
+
env['HTTP_CUSTOM_REFERER'] || env['HTTP_REFERER']
|
29
|
+
end
|
30
|
+
|
31
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'uses proc to evaluate referer from env' do
|
36
|
+
get '/', {},
|
37
|
+
'HTTP_CUSTOM_REFERER' => 'http://google.com?q=test',
|
38
|
+
'HTTP_REFERER' => 'http://bing.com?q=test1'
|
39
|
+
last_response[:referer].should be
|
40
|
+
last_response[:referer].search_engine.should == :google
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'uses proc to evaluate referer from env on every request' do
|
44
|
+
get '/', {}, 'HTTP_REFERER' => 'http://bing.com?q=test1'
|
45
|
+
last_response[:referer].should be
|
46
|
+
last_response[:referer].search_engine.should == :bing
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'default' do
|
51
|
+
let(:app) do
|
52
|
+
Rack::Builder.new do
|
53
|
+
use SearchEngineReferer::Middleware
|
54
|
+
run lambda{|env| [200, {:referer => env['search_engine_referer']}, ['OK']]}
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'uses HTTP_REFERER by ' do
|
59
|
+
get '/', {}, 'HTTP_REFERER' => 'http://google.com?q=test'
|
60
|
+
last_response[:referer].should be
|
61
|
+
last_response[:referer].search_engine.should == :google
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Rambler do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be rambler' do
|
7
|
+
parser.new('http://nova.rambler.ru').search_engine.should == :rambler
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it is not specified' do
|
13
|
+
parser.new('http://nova.rambler.ru?query=test').page.should == 1
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page if it is specified in url' do
|
17
|
+
parser.new('http://nova.rambler.ru?query=test&page=2').page.should == 2
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#query' do
|
22
|
+
it 'should parse page if it is not specified' do
|
23
|
+
parser.new('http://nova.rambler.ru?query=test').query.should == 'test'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer::Yandex do
|
4
|
+
let(:parser){described_class}
|
5
|
+
describe '#search_engine' do
|
6
|
+
it 'should be yandex' do
|
7
|
+
parser.new('http://yandex.ru').search_engine.should == :yandex
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#page' do
|
12
|
+
it 'should parse page if it specified in query' do
|
13
|
+
parser.new('http://yandex.ru?p=2').page.should == 3
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should parse page if it not specified in query' do
|
17
|
+
parser.new('http://yandex.ru').page.should == 1
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#query' do
|
22
|
+
it 'should parse user query correctly from text parameter' do
|
23
|
+
parser.new(URI.escape('http://yandex.ru?text=test user request')).
|
24
|
+
query.should == 'test user request'
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should parse user query with unicode symbols' do
|
28
|
+
parser.new(URI.escape('http://yandex.ru?text=тестовый запрос')).
|
29
|
+
query.should == 'тестовый запрос'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SearchEngineReferer do
|
4
|
+
describe '.engines' do
|
5
|
+
it 'should return all supported engines' do
|
6
|
+
SearchEngineReferer.engine_names.should =~ [:yandex, :mailru, :google, :rambler, :bing]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.parse' do
|
11
|
+
it 'should return referer or nil' do
|
12
|
+
SearchEngineReferer.parse('http://yandex.ru').
|
13
|
+
should be_kind_of(SearchEngineReferer::Base)
|
14
|
+
|
15
|
+
SearchEngineReferer.parse('http://test.ru').should be_nil
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should handle empty referer' do
|
19
|
+
SearchEngineReferer.parse(nil).should be_nil
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should handle known domains only' do
|
23
|
+
SearchEngineReferer.parse('http://test.ru?q=1&text=2').should be_nil
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'should handle google referer' do
|
27
|
+
SearchEngineReferer.parse('http://google.ru?q=1&text=2').query.should == '1'
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'should handle yandex referer' do
|
31
|
+
SearchEngineReferer.parse('http://yandex.ru?q=1&text=2').query.should == '2'
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should handle rambler referer' do
|
35
|
+
SearchEngineReferer.parse('http://nova.rambler.ru?query=1&page=2').query.should == '1'
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should unescape params found' do
|
39
|
+
SearchEngineReferer.parse('http://google.ru?q=1+2').query.should == '1 2'
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should handle urls without query params' do
|
43
|
+
SearchEngineReferer.parse('http://google.ru').query.should == ''
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should return nil on incorrect referer' do
|
47
|
+
SearchEngineReferer.parse('http://www.aviasales.ru/\\').should be_nil
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'should return nil on incorrect referer' do
|
51
|
+
SearchEngineReferer.parse('string with spaces').should be_nil
|
52
|
+
end
|
53
|
+
|
54
|
+
it 'should use iconv for cp1251 strings from yandex' do
|
55
|
+
SearchEngineReferer.parse('http://yandex.ru/yandsearch?text=%e0%e2%e8%e0%f1%e5%eb%f1').
|
56
|
+
query.should == 'авиаселс'
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'search_engine_referer'
|
metadata
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: search_engine_referer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alexander Stanko
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-12-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rspec
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activesupport
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rack-test
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rack
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: "\n Help to parse referer url for get user search query and page of
|
70
|
+
search engine results.\n Work with Google and Yandex for now.\n "
|
71
|
+
email:
|
72
|
+
- astanko@aviasales.ru
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- .gitignore
|
78
|
+
- .rspec
|
79
|
+
- .ruby-gemset
|
80
|
+
- .ruby-version
|
81
|
+
- Gemfile
|
82
|
+
- LICENSE
|
83
|
+
- README.md
|
84
|
+
- Rakefile
|
85
|
+
- lib/search_engine_referer.rb
|
86
|
+
- lib/search_engine_referer/base.rb
|
87
|
+
- lib/search_engine_referer/bing.rb
|
88
|
+
- lib/search_engine_referer/controller_helper.rb
|
89
|
+
- lib/search_engine_referer/google.rb
|
90
|
+
- lib/search_engine_referer/mailru.rb
|
91
|
+
- lib/search_engine_referer/middleware.rb
|
92
|
+
- lib/search_engine_referer/railtie.rb
|
93
|
+
- lib/search_engine_referer/rambler.rb
|
94
|
+
- lib/search_engine_referer/version.rb
|
95
|
+
- lib/search_engine_referer/yandex.rb
|
96
|
+
- search_engine_referer.gemspec
|
97
|
+
- spec/search_engine_referer/base_spec.rb
|
98
|
+
- spec/search_engine_referer/bing_spec.rb
|
99
|
+
- spec/search_engine_referer/controller_helper_spec.rb
|
100
|
+
- spec/search_engine_referer/google_spec.rb
|
101
|
+
- spec/search_engine_referer/mailru_spec.rb
|
102
|
+
- spec/search_engine_referer/middleware_spec.rb
|
103
|
+
- spec/search_engine_referer/rambler_spec.rb
|
104
|
+
- spec/search_engine_referer/yandex_spec.rb
|
105
|
+
- spec/search_engine_referer_spec.rb
|
106
|
+
- spec/spec_helper.rb
|
107
|
+
homepage: http://github.com/Rakoth/search_engine_referer
|
108
|
+
licenses: []
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project: search_engine_referer
|
126
|
+
rubygems_version: 2.0.3
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Parse search engine referer
|
130
|
+
test_files:
|
131
|
+
- spec/search_engine_referer/base_spec.rb
|
132
|
+
- spec/search_engine_referer/bing_spec.rb
|
133
|
+
- spec/search_engine_referer/controller_helper_spec.rb
|
134
|
+
- spec/search_engine_referer/google_spec.rb
|
135
|
+
- spec/search_engine_referer/mailru_spec.rb
|
136
|
+
- spec/search_engine_referer/middleware_spec.rb
|
137
|
+
- spec/search_engine_referer/rambler_spec.rb
|
138
|
+
- spec/search_engine_referer/yandex_spec.rb
|
139
|
+
- spec/search_engine_referer_spec.rb
|
140
|
+
- spec/spec_helper.rb
|