ozon_parser 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 45995c619b39e7412d06daa6c5fc97aaadc1e754
4
+ data.tar.gz: 399cfb8f8c04df165d098e8b002f0cc71c899b06
5
+ SHA512:
6
+ metadata.gz: 41d3f306e471150743f9c93d912b3bbb07d6b8724d2686a0258167c6d663344409b65a32534ca6c31f5aaa6e74bf5dddedf436ccf1a0ea235e97d918c5ce0e4b
7
+ data.tar.gz: 7d26342433525d7a47a80970f47856729ea2eb5d5c6f1c8a27eb8853973ff594950dfda2798bcef3a482cb5eb75a0c3142ae9449ed8e1c1d851b0ba8d3aebb7c
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ InstalledFiles
7
+ _yardoc
8
+ coverage
9
+ doc/
10
+ lib/bundler/man
11
+ pkg
12
+ rdoc
13
+ spec/reports
14
+ test/tmp
15
+ test/version_tmp
16
+ tmp
17
+ .idea
18
+ test/cassettes
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in ozon_parser.gemspec
4
+ gemspec
@@ -0,0 +1,86 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ ozon_parser (0.0.1)
5
+ activesupport
6
+ mechanize (= 2.7.1)
7
+ multi_json (~> 1.9)
8
+ nokogiri (~> 1.6.1)
9
+
10
+ GEM
11
+ remote: https://rubygems.org/
12
+ specs:
13
+ activesupport (3.2.17)
14
+ i18n (~> 0.6, >= 0.6.4)
15
+ multi_json (~> 1.0)
16
+ addressable (2.3.5)
17
+ crack (0.4.2)
18
+ safe_yaml (~> 1.0.0)
19
+ database_cleaner (1.2.0)
20
+ diff-lcs (1.2.5)
21
+ domain_name (0.5.16)
22
+ unf (>= 0.0.5, < 1.0.0)
23
+ http-cookie (1.0.2)
24
+ domain_name (~> 0.5)
25
+ i18n (0.6.9)
26
+ mechanize (2.7.1)
27
+ domain_name (~> 0.5, >= 0.5.1)
28
+ http-cookie (~> 1.0.0)
29
+ mime-types (~> 1.17, >= 1.17.2)
30
+ net-http-digest_auth (~> 1.1, >= 1.1.1)
31
+ net-http-persistent (~> 2.5, >= 2.5.2)
32
+ nokogiri (~> 1.4)
33
+ ntlm-http (~> 0.1, >= 0.1.1)
34
+ webrobots (>= 0.0.9, < 0.2)
35
+ mime-types (1.25.1)
36
+ mini_portile (0.5.2)
37
+ minispec-metadata (2.0.0)
38
+ minitest
39
+ minitest (5.3.0)
40
+ minitest-vcr (0.1.0)
41
+ database_cleaner (>= 1.0.1)
42
+ minispec-metadata (~> 2.0.0)
43
+ minitest (>= 4.7.5)
44
+ vcr (~> 2.8.0)
45
+ multi_json (1.9.0)
46
+ net-http-digest_auth (1.4)
47
+ net-http-persistent (2.9.4)
48
+ nokogiri (1.6.1)
49
+ mini_portile (~> 0.5.0)
50
+ ntlm-http (0.1.1)
51
+ predicated (0.2.6)
52
+ rake (10.1.1)
53
+ ruby2ruby (2.0.7)
54
+ ruby_parser (~> 3.1)
55
+ sexp_processor (~> 4.0)
56
+ ruby_parser (3.2.2)
57
+ sexp_processor (~> 4.1)
58
+ safe_yaml (1.0.1)
59
+ sexp_processor (4.4.1)
60
+ unf (0.1.3)
61
+ unf_ext
62
+ unf_ext (0.0.6)
63
+ vcr (2.8.0)
64
+ webmock (1.11.0)
65
+ addressable (>= 2.2.7)
66
+ crack (>= 0.3.2)
67
+ webrobots (0.1.1)
68
+ wrong (0.7.1)
69
+ diff-lcs (~> 1.2.5)
70
+ predicated (~> 0.2.6)
71
+ ruby2ruby (>= 2.0.1)
72
+ ruby_parser (>= 3.0.1)
73
+ sexp_processor (>= 4.0)
74
+
75
+ PLATFORMS
76
+ ruby
77
+
78
+ DEPENDENCIES
79
+ bundler
80
+ minitest (~> 5.3.0)
81
+ minitest-vcr
82
+ ozon_parser!
83
+ rake
84
+ vcr
85
+ webmock (< 1.16)
86
+ wrong
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Artem Petrov
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,55 @@
1
+ #OzonParser
2
+
3
+ Компактный парсер ozon.ru (http://www.ozon.ru/)
4
+
5
+ ## Установка
6
+
7
+ Добавьте в ваш гемфайл строку
8
+
9
+ gem 'ozon_parser'
10
+
11
+ Затем выполните:
12
+
13
+ $ bundle
14
+
15
+ ## Использование
16
+
17
+ На текущий момент парсер корректно работает только с книгами.
18
+
19
+ ```
20
+ require 'ozon_parser'
21
+
22
+ @book = OzonParser::Pages::Book.new(url: "http://www.ozon.ru/context/detail/id/10422848/")
23
+
24
+ @book = OzonParser::Pages::Book.new(id: '7360633')
25
+ ```
26
+
27
+ Доступны методы:
28
+
29
+ + title - название
30
+ + type - тип продукта - пока что только книга
31
+ + cover_url - ссылка на картинку
32
+ + cover_file - сама картинка
33
+ + tags - тэги, взятые из хлебных крошек
34
+ + author - автор
35
+ + language - массив языков
36
+ + isbn - массив isbn
37
+ + pages - количество страниц
38
+
39
+ Более подробно смотрите в тестах
40
+
41
+
42
+ Тесты
43
+ -----
44
+
45
+ ```
46
+ rake
47
+ ```
48
+
49
+ ## Contributing
50
+
51
+ 1. Fork it
52
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
53
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
54
+ 4. Push to the branch (`git push origin my-new-feature`)
55
+ 5. Create new Pull Request
@@ -0,0 +1,9 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new do |t|
5
+ t.libs << "test"
6
+ t.pattern = 'test/**/*_test.rb'
7
+ end
8
+
9
+ task :default => :test
@@ -0,0 +1,47 @@
1
+ require 'active_support/core_ext'
2
+ require 'mechanize'
3
+ require 'multi_json'
4
+ require 'round_robin'
5
+
6
+ require 'ozon_parser/version'
7
+ require 'ozon_parser/config'
8
+ require 'ozon_parser/agent'
9
+ require 'ozon_parser/proxy'
10
+ require 'ozon_parser/proxy_list'
11
+
12
+ require 'ozon_parser/page'
13
+ require 'ozon_parser/pages/base'
14
+ require 'ozon_parser/pages/book'
15
+
16
+ module OzonParser
17
+ Encoding.default_external = Encoding::UTF_8
18
+
19
+ def self.configure
20
+ @config = OzonParser::Config.new
21
+ yield @config
22
+ end
23
+
24
+ def self.config
25
+ @config ||= OzonParser::Config.new
26
+ end
27
+
28
+ def self.config_reset
29
+ @config = OzonParser::Config.new
30
+ end
31
+
32
+ class BaseError < StandardError
33
+ end
34
+
35
+ class TypeError < BaseError
36
+ end
37
+
38
+ class DecodeError < BaseError
39
+ end
40
+
41
+ class ConnectionError < BaseError
42
+ end
43
+
44
+ class InitError < BaseError
45
+ end
46
+
47
+ end
@@ -0,0 +1,87 @@
1
+ module OzonParser
2
+ class Agent
3
+ include RoundRobin
4
+
5
+ AGENT_ALIASES = ['Mac Safari', 'Mac Firefox', 'Linux Firefox', 'Linux Mozilla',
6
+ 'Windows IE 8', 'Windows IE 9', 'Windows Mozilla']
7
+
8
+ attr_reader :current_proxy, :options
9
+
10
+ def initialize(options = {})
11
+ @agent = Mechanize.new { |agent|
12
+ agent.open_timeout = OzonParser.config.open_timeout if OzonParser.config.open_timeout
13
+ agent.read_timeout = OzonParser.config.read_timeout if OzonParser.config.read_timeout
14
+ }
15
+ @options = options
16
+ @proxy = Proxy.new(options[:proxy]) if options[:proxy]
17
+ end
18
+
19
+ def history
20
+ @agent.history
21
+ end
22
+
23
+ def get(url, parameters = [], referer = nil, headers = {})
24
+ @agent.reset
25
+ @agent.follow_redirect = false
26
+
27
+ set_proxy
28
+ set_user_agent
29
+
30
+ begin
31
+ @page = @agent.get(url, [], referer)
32
+ raise if @page.code == '302'
33
+ rescue => e
34
+ p e
35
+ freeze_current_proxy
36
+ raise e
37
+ end
38
+
39
+ @page.body
40
+ end
41
+
42
+ def get_file(url)
43
+ begin
44
+ file = @agent.get url
45
+ rescue => e
46
+ freeze_current_proxy
47
+ raise e
48
+ end
49
+
50
+ ext = file.filename.split('.').last
51
+ if file.body_io.instance_of?(Tempfile)
52
+ body = StringIO.new(file.body_io.read)
53
+ create_temp(file.filename, body, ext)
54
+ else
55
+ create_temp(file.filename, file.body_io, ext)
56
+ end
57
+ end
58
+
59
+ def create_temp(name, body, ext)
60
+ temp_file = Tempfile.new([name, ".#{ext}"])
61
+ temp_file.binmode
62
+ temp_file.write body.string
63
+ temp_file.flush
64
+ temp_file
65
+ end
66
+
67
+ def set_proxy
68
+ return if @options[:proxy] === false
69
+
70
+ return unless @proxy || OzonParser.config.use_proxy?
71
+
72
+ @current_proxy = @proxy || OzonParser.config.proxy_list.next
73
+ @agent.set_proxy current_proxy.host, current_proxy.port
74
+ end
75
+
76
+ def set_user_agent
77
+ @agent.user_agent_alias = next! AGENT_ALIASES
78
+ end
79
+
80
+ private
81
+
82
+ def freeze_current_proxy
83
+ OzonParser.config.proxy_list.freeze_proxy current_proxy
84
+ end
85
+
86
+ end
87
+ end
@@ -0,0 +1,19 @@
1
+ module OzonParser
2
+ class Config
3
+ attr_accessor :proxy_list, :proxy_freeze_time, :read_timeout, :open_timeout
4
+
5
+ def proxy_list=(url_list)
6
+ @proxy_list = OzonParser::ProxyList.new
7
+
8
+ url_list.each do |url|
9
+ proxy = Proxy.new url
10
+ @proxy_list << proxy
11
+ end
12
+ end
13
+
14
+ def use_proxy?
15
+ proxy_list && proxy_list.any?
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,81 @@
1
+ # encoding: utf-8
2
+ module OzonParser
3
+
4
+ class Page
5
+ # < Hash
6
+ # include Hashie::Extensions::MethodAccess
7
+
8
+ BASE_OZON_URL = 'http://www.ozon.ru'
9
+
10
+ attr_accessor :result
11
+
12
+ def self.field(name, &block)
13
+ if block_given?
14
+ attr_writer name.to_sym
15
+ send :define_method, name.to_sym do
16
+ unless instance_variable_defined?("@#{name}")
17
+ value = instance_eval(&block)
18
+ value = clean_up(value) if value.is_a?(String)
19
+ instance_variable_set("@#{name}", value)
20
+ end
21
+ instance_variable_get("@#{name}")
22
+ end
23
+ else
24
+ attr_accessor name.to_sym
25
+ end
26
+ end
27
+
28
+ # @param [Object] options
29
+ # @option options [String] :source Сырой HTML в виде строки
30
+ # @option options [String] :url Урла
31
+ def initialize(options = {})
32
+ @source = options.delete(:source)
33
+ @url = options.delete(:url)
34
+ end
35
+
36
+ def agent
37
+ options = {
38
+ proxy: @proxy
39
+ }
40
+
41
+ @agent ||= OzonParser::Agent.new options
42
+ end
43
+
44
+ def source
45
+ @source ||= agent.get(@url) || throw('Define source or url... =(')
46
+ rescue Exception => e
47
+ raise ConnectionError, "#{e.to_s}, #{agent.history.inspect}", e.backtrace
48
+ end
49
+
50
+ def json
51
+ return source if source.is_a?(Hash)
52
+ @json ||= MultiJson.decode source if source
53
+ rescue Exception => e
54
+ raise DecodeError, "#{e.to_s}, #{agent.history.inspect}", e.backtrace
55
+ end
56
+
57
+ def root
58
+ @root ||= Nokogiri::HTML(source, 'utf-8')
59
+ end
60
+
61
+ def file(file_url)
62
+ agent.get_file file_url
63
+ end
64
+
65
+ def clean_up(str)
66
+ str.strip.sub(/^\.{3}/, '').sub(/\,$/, '').gsub("\u00A0", "\u0020").strip
67
+ end
68
+
69
+ def untypofy(str)
70
+ str.gsub("\u0097", "—")
71
+ end
72
+
73
+ def format_date(str)
74
+ ['января', 'февраля', 'марта', 'апреля',
75
+ 'мая', 'июня', 'июля', 'августа', 'сентября',
76
+ 'октября', 'ноября', 'декабря'].each_with_index { |month, i| str = str.sub(month, (i + 1).to_s) }
77
+ str.strip.gsub(/[^\d]+/, '-')
78
+ end
79
+
80
+ end
81
+ end
@@ -0,0 +1,31 @@
1
+ module OzonParser
2
+ module Pages
3
+ class Base < OzonParser::Page
4
+ attr_accessor :id, :url
5
+
6
+
7
+ # This method need to point url source of page
8
+ # @return [String] url
9
+ def parse_url(id)
10
+ BASE_OZON_URL + "/context/detail/id/#{id}/"
11
+ end
12
+
13
+ def initialize(args = {})
14
+
15
+ @source = args.delete(:source)
16
+ @url= args.delete(:url)
17
+ @id= args.delete(:id)
18
+
19
+ args.each do |k,v|
20
+ instance_variable_set("@#{k}", v) unless v.nil?
21
+ end
22
+
23
+ return super(source: @source) if @source
24
+ return super(url: @url) if @url
25
+ raise InitError if @id.nil?
26
+ @url = parse_url(@id)
27
+ super(url: @url)
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,64 @@
1
+ module OzonParser
2
+ module Pages
3
+ class Book < Base
4
+
5
+ field :categories do
6
+ cats = root.xpath('//ul[@class="navLine"]/li')
7
+ cats.map &:text
8
+ end
9
+
10
+ field :title do
11
+ categories.last
12
+ end
13
+
14
+ field :type do
15
+ categories.first
16
+ end
17
+
18
+ field :cover_url do
19
+ dirty = root.xpath('//img[@class="eMicroGallery_fullImage"]/@src').try(:text)
20
+ if dirty
21
+ clean = dirty.split('/').map{|x| x if x.present? && x != "c300"}.compact.join("/")
22
+ "http://" + clean
23
+ end
24
+ end
25
+
26
+ field :cover_file do
27
+ try(:file, cover_url)
28
+ end
29
+
30
+ field :tags do
31
+ categories - [title, type]
32
+ end
33
+
34
+ field :author do
35
+ dirty = root.xpath('//div[@class="bGeneralProperties"]/p[@itemprop="author"]/a/@title').try(:text)
36
+ end
37
+
38
+ field :language do
39
+ dirty = root.xpath('//div[@class="bGeneralProperties"]/p[@itemprop="inLanguage"]').try(:text)
40
+ if dirty
41
+ clean = dirty.gsub("Языки:","")
42
+ clean.split(",").map &:strip
43
+ end
44
+ end
45
+
46
+ field :isbn do
47
+ dirty = root.xpath('//div[@class="bGeneralProperties"]/p[@itemprop="isbn"]').try(:text)
48
+ if dirty
49
+ numbers_with_year = dirty.gsub("ISBN ", "")
50
+ numbers = numbers_with_year.split(";").first
51
+ numbers.split(",").map &:strip
52
+ end
53
+ end
54
+
55
+ field :pages do
56
+ dirty = root.xpath('//div[@class="bTechCover "]/div[@class="bTechDescr"]/span[@itemprop="numberOfPages"]').try(:text)
57
+ if dirty
58
+ dirty.to_i
59
+ end
60
+ end
61
+
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,22 @@
1
+ module OzonParser
2
+ class Proxy
3
+ attr_reader :host, :port
4
+
5
+ def initialize(url)
6
+ uri = URI url
7
+ @host, @port = uri.host, uri.port
8
+ end
9
+
10
+ def freeze
11
+ @frozen_to = Time.now + freeze_time
12
+ end
13
+
14
+ def freeze_time
15
+ OzonParser.config.proxy_freeze_time || 1.minute
16
+ end
17
+
18
+ def frozen?
19
+ Time.now < @frozen_to
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,42 @@
1
+ module OzonParser
2
+ class ProxyList
3
+ include Enumerable
4
+ include RoundRobin
5
+
6
+ attr_reader :frozen_proxies
7
+ attr_reader :proxies
8
+
9
+ def initialize(proxies = [])
10
+ @proxies = proxies
11
+ @frozen_proxies = []
12
+ end
13
+
14
+ def each(&block)
15
+ @proxies.each(&block)
16
+ end
17
+
18
+ def <<(proxy)
19
+ @proxies << proxy
20
+ end
21
+
22
+ def next
23
+ unfreeze_proxies
24
+ next! @proxies
25
+ end
26
+
27
+ def freeze_proxy(proxy)
28
+ return unless proxy
29
+ @proxies.delete proxy
30
+ proxy.freeze
31
+ @frozen_proxies << proxy unless proxy.in?(@frozen_proxies)
32
+ end
33
+
34
+ def unfreeze_proxies
35
+ @frozen_proxies.each do |proxy|
36
+ return if proxy.frozen?
37
+ @frozen_proxies.delete proxy
38
+ @proxies << proxy
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,3 @@
1
+ module OzonParser
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,7 @@
1
+ module RoundRobin
2
+ def next!(arry)
3
+ value = arry.shift
4
+ arry << value
5
+ value
6
+ end
7
+ end
@@ -0,0 +1,34 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'ozon_parser/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "ozon_parser"
8
+ spec.version = OzonParser::VERSION
9
+ spec.authors = ["Partos"]
10
+ spec.email = ["partos0511@gmail.com"]
11
+ spec.description = %q{Simple OZON parser}
12
+ spec.summary = %q{Simple OZON parser}
13
+ spec.homepage = ""
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_runtime_dependency "nokogiri", "~> 1.6.1"
22
+ spec.add_runtime_dependency "mechanize", "2.7.1"
23
+ spec.add_runtime_dependency "activesupport"
24
+ spec.add_runtime_dependency "multi_json", "~> 1.9"
25
+
26
+ spec.add_development_dependency "bundler"
27
+ spec.add_development_dependency "rake"
28
+
29
+ spec.add_development_dependency 'minitest', "~> 5.3.0"
30
+ spec.add_development_dependency 'minitest-vcr'
31
+ spec.add_development_dependency 'vcr'
32
+ spec.add_development_dependency 'webmock', '< 1.16'
33
+ spec.add_development_dependency 'wrong'
34
+ end
@@ -0,0 +1,43 @@
1
+ require 'test_helper'
2
+
3
+ class AgentTest < TestCase
4
+
5
+ def setup
6
+ end
7
+
8
+ def teardown
9
+ OzonParser.config_reset
10
+ end
11
+
12
+ def test_agent_with_proxy
13
+ agent = OzonParser::Agent.new proxy: 'http://127.0.0.1:8080'
14
+ agent.set_proxy
15
+
16
+ assert { agent.current_proxy.host == '127.0.0.1'}
17
+ assert { agent.current_proxy.port == 8080 }
18
+ end
19
+
20
+
21
+ def test_configure_method
22
+ OzonParser.configure do |config|
23
+ config.proxy_list = ['http://127.0.0.2:8081']
24
+ end
25
+
26
+ agent = OzonParser::Agent.new
27
+ agent.set_proxy
28
+
29
+ assert { agent.current_proxy.host == '127.0.0.2' }
30
+ assert { agent.current_proxy.port == 8081 }
31
+ end
32
+
33
+ def test_configure_method_with_proxy_false
34
+ OzonParser.configure do |config|
35
+ config.proxy_list = ['http://127.0.0.3:8082']
36
+ end
37
+
38
+ agent = OzonParser::Agent.new proxy: false
39
+ agent.set_proxy
40
+
41
+ assert { agent.current_proxy.nil? }
42
+ end
43
+ end
@@ -0,0 +1,79 @@
1
+ # encoding: utf-8
2
+ require 'test_helper'
3
+
4
+ class BookPageTest < TestCase
5
+
6
+ def setup
7
+ VCR.insert_cassette 'book'
8
+ @book = OzonParser::Pages::Book.new(id: '7360633')
9
+ @book2 = OzonParser::Pages::Book.new(url: "http://www.ozon.ru/context/detail/id/10422848/")
10
+ end
11
+
12
+ def teardown
13
+ VCR.eject_cassette
14
+ end
15
+
16
+ def test_correct_url
17
+ assert_equal @book.url, "http://www.ozon.ru/context/detail/id/7360633/"
18
+ assert_equal @book2.url, "http://www.ozon.ru/context/detail/id/10422848/"
19
+ end
20
+
21
+ def test_correct_title
22
+ assert_equal @book.title, "Идеальный программист. Как стать профессионалом разработки ПО"
23
+ assert_equal @book2.title, "Дикая природа России / Wildlife in Russia"
24
+ end
25
+
26
+ def test_cover_url
27
+ assert_equal @book.cover_url, "http://static1.ozone.ru/multimedia/books_covers/1003645102.jpg"
28
+ assert_equal @book2.cover_url, "http://static.ozone.ru/multimedia/books_covers/1005266671.jpg"
29
+ end
30
+
31
+ def test_categories
32
+ assert_equal @book.categories, ["Книги", "Нехудожественная литература",
33
+ "Компьютерная литература",
34
+ "Разработка программного обеспечения",
35
+ "Алгоритмы и методы. Искусство программирования",
36
+ "Идеальный программист. Как стать профессионалом разработки ПО"]
37
+
38
+ assert_equal @book2.categories, ["Книги", "Нехудожественная литература",
39
+ "Научная и техническая литература", "Естественные науки", "Биологические науки",
40
+ "Общая биология. Палеонтология", "Дикая природа России / Wildlife in Russia"]
41
+ end
42
+
43
+ def test_type
44
+ assert_equal @book.type, "Книги"
45
+ assert_equal @book2.type, "Книги"
46
+ end
47
+
48
+ def test_tags
49
+ assert_equal @book.tags, ["Нехудожественная литература",
50
+ "Компьютерная литература",
51
+ "Разработка программного обеспечения",
52
+ "Алгоритмы и методы. Искусство программирования"]
53
+
54
+ assert_equal @book2.tags, ["Нехудожественная литература",
55
+ "Научная и техническая литература", "Естественные науки", "Биологические науки",
56
+ "Общая биология. Палеонтология"]
57
+ end
58
+
59
+ def test_author
60
+ assert_equal @book.author, "Роберт К. Мартин"
61
+ assert_equal @book2.author, "Валерий Малеев"
62
+ end
63
+
64
+ def test_languages
65
+ assert_equal @book.language, ["Русский"]
66
+ assert_equal @book2.language, ["Русский", "Английский"]
67
+ end
68
+
69
+ def test_isbn
70
+ assert_equal @book.isbn, ["978-5-459-01044-2","978-0137081073"]
71
+ assert_equal @book2.isbn, ["978-5-91657-436-4"]
72
+ end
73
+
74
+ def test_pages
75
+ assert_equal @book.pages, 224
76
+ assert_equal @book2.pages, 496
77
+ end
78
+
79
+ end
@@ -0,0 +1,26 @@
1
+ require 'test_helper'
2
+
3
+ class ProxyListTest < TestCase
4
+
5
+ def setup
6
+ @list = OzonParser::ProxyList.new ['http://127.0.0.1:8080', 'http://10.0.0.1:3040']
7
+ end
8
+
9
+ def teardown
10
+ OzonParser.config_reset
11
+ end
12
+
13
+ def test_round_robin
14
+ first_proxy = @list.next
15
+ second_proxy = @list.next
16
+
17
+ assert { first_proxy != second_proxy }
18
+ end
19
+
20
+ def test_freeze_proxy
21
+ proxy = @list.next
22
+ @list.freeze_proxy proxy
23
+ deny { @list.include? proxy }
24
+ end
25
+
26
+ end
@@ -0,0 +1,23 @@
1
+ require 'test_helper'
2
+
3
+ class ProxyTest < TestCase
4
+
5
+ def setup
6
+ @proxy = OzonParser::Proxy.new 'http://127.0.0.1:8080'
7
+ end
8
+
9
+ def teardown
10
+ OzonParser.config_reset
11
+ end
12
+
13
+ def test_proxy_has_port_and_host
14
+ assert { @proxy.host == '127.0.0.1' }
15
+ assert { @proxy.port == 8080 }
16
+ end
17
+
18
+ def test_freeze_proxy
19
+ @proxy.freeze
20
+ assert { @proxy.frozen? }
21
+ end
22
+
23
+ end
@@ -0,0 +1,32 @@
1
+ require 'bundler/setup'
2
+ Bundler.require
3
+
4
+ require "minitest/autorun"
5
+ require "minitest/pride"
6
+ require "vcr"
7
+ # require "minitest-vcr"
8
+ require "webmock"
9
+ require 'webmock/minitest'
10
+
11
+ require 'wrong'
12
+ require 'wrong/adapters/minitest'
13
+ Wrong.config.color
14
+ # require "mocha/setup"
15
+
16
+ VCR.configure do |c|
17
+ # c.allow_http_connections_when_no_cassette = true
18
+ c.cassette_library_dir = 'test/cassettes'
19
+ c.hook_into :webmock
20
+ end
21
+
22
+ # MinitestVcr::Spec.configure!
23
+
24
+ class TestCase < Minitest::Test
25
+ include Wrong
26
+ include WebMock::API
27
+ WebMock.disable_net_connect!
28
+ end
29
+
30
+
31
+
32
+
metadata ADDED
@@ -0,0 +1,226 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ozon_parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Partos
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-03-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: nokogiri
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: 1.6.1
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 1.6.1
27
+ - !ruby/object:Gem::Dependency
28
+ name: mechanize
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '='
32
+ - !ruby/object:Gem::Version
33
+ version: 2.7.1
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '='
39
+ - !ruby/object:Gem::Version
40
+ version: 2.7.1
41
+ - !ruby/object:Gem::Dependency
42
+ name: activesupport
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: multi_json
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: '1.9'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: '1.9'
69
+ - !ruby/object:Gem::Dependency
70
+ name: bundler
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rake
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '>='
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: minitest
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: 5.3.0
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ~>
109
+ - !ruby/object:Gem::Version
110
+ version: 5.3.0
111
+ - !ruby/object:Gem::Dependency
112
+ name: minitest-vcr
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - '>='
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: vcr
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - '>='
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - '>='
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: webmock
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - <
144
+ - !ruby/object:Gem::Version
145
+ version: '1.16'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - <
151
+ - !ruby/object:Gem::Version
152
+ version: '1.16'
153
+ - !ruby/object:Gem::Dependency
154
+ name: wrong
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - '>='
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - '>='
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ description: Simple OZON parser
168
+ email:
169
+ - partos0511@gmail.com
170
+ executables: []
171
+ extensions: []
172
+ extra_rdoc_files: []
173
+ files:
174
+ - .gitignore
175
+ - Gemfile
176
+ - Gemfile.lock
177
+ - LICENSE.txt
178
+ - README.md
179
+ - Rakefile
180
+ - lib/ozon_parser.rb
181
+ - lib/ozon_parser/agent.rb
182
+ - lib/ozon_parser/config.rb
183
+ - lib/ozon_parser/page.rb
184
+ - lib/ozon_parser/pages/base.rb
185
+ - lib/ozon_parser/pages/book.rb
186
+ - lib/ozon_parser/proxy.rb
187
+ - lib/ozon_parser/proxy_list.rb
188
+ - lib/ozon_parser/version.rb
189
+ - lib/round_robin.rb
190
+ - ozon_parser.gemspec
191
+ - test/lib/ozon_parser/agent_test.rb
192
+ - test/lib/ozon_parser/pages/book_test.rb
193
+ - test/lib/ozon_parser/proxy_list_test.rb
194
+ - test/lib/ozon_parser/proxy_test.rb
195
+ - test/test_helper.rb
196
+ homepage: ''
197
+ licenses:
198
+ - MIT
199
+ metadata: {}
200
+ post_install_message:
201
+ rdoc_options: []
202
+ require_paths:
203
+ - lib
204
+ required_ruby_version: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - '>='
207
+ - !ruby/object:Gem::Version
208
+ version: '0'
209
+ required_rubygems_version: !ruby/object:Gem::Requirement
210
+ requirements:
211
+ - - '>='
212
+ - !ruby/object:Gem::Version
213
+ version: '0'
214
+ requirements: []
215
+ rubyforge_project:
216
+ rubygems_version: 2.0.3
217
+ signing_key:
218
+ specification_version: 4
219
+ summary: Simple OZON parser
220
+ test_files:
221
+ - test/lib/ozon_parser/agent_test.rb
222
+ - test/lib/ozon_parser/pages/book_test.rb
223
+ - test/lib/ozon_parser/proxy_list_test.rb
224
+ - test/lib/ozon_parser/proxy_test.rb
225
+ - test/test_helper.rb
226
+ has_rdoc: