apress-yandex_market 0.1.0

data/lib/apress/yandex_market/readers/model_by_category.rb

@@ -0,0 +1,145 @@
+require 'apress/yandex_market/readers/base'
+require 'apress/yandex_market/readers/category'
+require 'apress/yandex_market/presenters'
+
+module Apress
+  module YandexMarket
+    module Readers
+      # Ридер моделей Яндекс.Маркета по категориям
+      #
+      # Examples
+      #
+      #   reader = Apress::YandexMarket::Readers::ModelByCategory.new(region_id: 225,
+      #                                                               token: 'qwerty',
+      #                                                               categories: 'Авто; Дом и дача')
+      #   # => #<Apress::YandexMarket::Readers::ModelByCategory ...>
+      #
+      #   reader.each_row |model|
+      #     puts model
+      #   end
+      #   # => {
+      #          :__line__=>1,
+      #          :__column__=>1,
+      #          :id=>217189255,
+      #          :name=>"Caviale Крем для лица Витамин Е",
+      #          :link=>"https://market.yandex.ru/product/217189255?hid=8476099&pp=929&clid=2326601&distr_type=7",
+      #          :description=>"время нанесения: дневной/ночной, эффект: увлажнение",
+      #          :photo=>{:url=>"https://avatars.mds.yandex.net/get-mpic/331398/img_id833813452766046986.jpeg/orig"},
+      #          :photos=>[
+      #            {:url=>"https://avatars.mds.yandex.net/get-mpic/331398/img_id833813452766046986.jpeg/orig"},
+      #            {:url=>"https://avatars.mds.yandex.net/get-mpic/331398/img_id8176844263482482207.jpeg/orig"},
+      #            {:url=>"https://avatars.mds.yandex.net/get-mpic/933699/img_id2264997491683174997.jpeg/orig"}
+      #          ],
+      #          :price=>{:min=>"68"},
+      #          :vendor=>{:name=>"Caviale"},
+      #          :specification=>[
+      #            {
+      #              :features=>[
+      #                {:value=>"время нанесения: дневной/ночной"},
+      #                {:value=>"эффект: увлажнение"}
+      #              ]
+      #            }
+      #          ]
+      #        }
+      #        {
+      #          :__line__=>2,
+      #          :__column__=>1,
+      #          :id=>217189256,
+      #          ...
+      #        }
+      class ModelByCategory < Base
+        FIELDS = %w(
+          MODEL_CATEGORY
+          MODEL_PHOTO
+          MODEL_PHOTOS
+          MODEL_PRICE
+          MODEL_SPECIFICATION
+          MODEL_VENDOR
+        ).join(',').freeze
+
+        class << self
+          def allowed_options
+            Readers::Category.allowed_options
+          end
+        end
+
+        attr_reader :category_reader
+
+        # Public: инициализация ридера
+        #
+        # options - Hash, параметры ридера
+        #   :region_id  - идентификатор региона (необязательный)
+        #   :token      - токен для доступа к API Яндекс.Маркета
+        #   :categories - список категорий для загрузки моделей Яндекс.Маркета
+        #
+        # Returns an instance of Readers::ModelByCategory
+        def initialize(options)
+          super
+          @category_reader = Readers::Category.new(options)
+        end
+
+        # Public: читаем товары из API и фильтруем необходимые поля для использования далее в препроцессоре Кирби
+        #
+        # Returns nothing
+        def each_row
+          presenter = Presenters::Model.new
+
+          category_reader.each_row do |category|
+            models = Set.new
+
+            begin
+              process_models(models, category.fetch(:id), 'DESC')
+            rescue Api::PageError
+              begin
+                process_models(models, category.fetch(:id), 'ASC')
+              rescue Api::PageError
+              end
+            end
+
+            models.each { |model| yield presenter.expose(model) }
+          end
+        end
+
+        private
+
+        def process_models(processed_models, category_id, sort_direction)
+          page = 1
+
+          loop do
+            models = get_models(category_id, page, sort_direction)
+
+            models.each do |model|
+              next if category_id != model.fetch(:category).fetch(:id)
+              return if processed_models.include? model
+
+              processed_models << model
+            end
+
+            break if models.size < PAGE_SIZE
+
+            page += 1
+          end
+        end
+
+        def get_models(category_id, page, sort_direction)
+          sleep(SLEEP_TIME)
+
+          with_rescue_temporary_errors do
+            client.
+              get(
+                "categories/#{category_id}/search",
+                geo_id: @region_id,
+                result_type: 'MODELS'.freeze,
+                sort: 'DATE'.freeze,
+                how: sort_direction,
+                fields: FIELDS,
+                count: PAGE_SIZE,
+                page: page
+              ).
+              fetch(:items)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/readers.rb

@@ -0,0 +1,8 @@
+module Apress
+  module YandexMarket
+    module Readers
+      autoload :Category, 'apress/yandex_market/readers/category'
+      autoload :ModelByCategory, 'apress/yandex_market/readers/model_by_category'
+    end
+  end
+end

data/lib/apress/yandex_market/version.rb

@@ -0,0 +1,5 @@
+module Apress
+  module YandexMarket
+    VERSION = '0.1.0'.freeze
+  end
+end

data/lib/apress/yandex_market.rb

@@ -0,0 +1,9 @@
+require 'apress/yandex_market/version'
+
+module Apress
+  module YandexMarket
+    autoload :Api, 'apress/yandex_market/api'
+    autoload :Presenters, 'apress/yandex_market/presenters'
+    autoload :Readers, 'apress/yandex_market/readers'
+  end
+end

data/spec/apress/yandex_market/api/client_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe Apress::YandexMarket::Api::Client do
+  let(:client) { described_class.new 'some_secret_token' }
+
+  describe '#get' do
+    context 'get root categores' do
+      subject { client.get('categories', geo_id: 225, count: 30) }
+
+      it do
+        VCR.use_cassette 'get_root_categories' do
+          expect(subject).to be_a Hash
+          expect(subject).to include(:categories, :context, :status)
+          expect(subject[:status]).to eq 'OK'
+          expect(subject[:categories]).to have(18).items
+        end
+      end
+    end
+
+    context 'when error' do
+      subject { client.get('categories/999999999', geo_id: 225) }
+
+      it do
+        VCR.use_cassette 'get_category_with_invalid_id' do
+          expect { subject }.to raise_error(Apress::YandexMarket::Api::Error, '404 - Category 999999999 not found')
+        end
+      end
+    end
+
+    context 'when error with html body' do
+      let(:error_body) do
+        '<html><body><h1>502 Bad Gateway</h1>The server returned an invalid or incomplete response.</body></html>'
+      end
+
+      subject { client.get('example') }
+
+      before do
+        stub_request(:get, 'https://api.content.market.yandex.ru/v2/example').to_return(status: 502, body: error_body)
+      end
+
+      it do
+        expect { VCR.turned_off { subject } }.to raise_error(Apress::YandexMarket::Api::Error, '502 - ')
+      end
+    end
+  end
+end

data/spec/apress/yandex_market/readers/category_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe Apress::YandexMarket::Readers::Category do
+  let(:categories) { 'Авто; Красота' }
+  let(:reader) { described_class.new(token: 'secret_token', categories: categories) }
+
+  describe '.allowed_options' do
+    it { expect(described_class.allowed_options).to eq %i(token region_id categories) }
+  end
+
+  describe '#each_row' do
+    let(:rows) { [] }
+
+    context 'when no errors' do
+      before do
+        VCR.use_cassette 'read_two_categories' do
+          reader.each_row { |row| rows << row }
+        end
+      end
+
+      it 'reads specified categories and their subcategories' do
+        expect(rows).to have(326).items
+        expect(rows.map { |row| row[:name] }).to include('Авто', 'Красота') # корневые категории
+        expect(rows.map { |row| row[:name] }).
+          to include('Авто', 'Запчасти', 'Фильтры', 'Воздушные фильтры') # поддерево от корня до листа
+      end
+    end
+
+    context 'when page error' do
+      before do
+        allow(reader.client).to receive(:get).and_call_original
+        allow(reader.client).to receive(:get).with('categories/90509/children', anything).
+          and_raise(Apress::YandexMarket::Api::PageError)
+
+        VCR.use_cassette 'read_two_categories_with_page_error' do
+          reader.each_row { |row| rows << row }
+        end
+      end
+
+      it 'reads specified categories and their subcategories' do
+        expect(rows).to have(187).items
+        expect(rows.map { |row| row[:name] }).to include('Авто', 'Красота')
+      end
+    end
+  end
+end

data/spec/apress/yandex_market/readers/model_by_category_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Apress::YandexMarket::Readers::ModelByCategory do
+  let(:categories) { 'Красота' } # id: 90509
+  let(:reader) { described_class.new(token: 'secret_token', categories: categories) }
+
+  describe '.allowed_options' do
+    it { expect(described_class.allowed_options).to eq %i(token region_id categories) }
+  end
+
+  describe '#each_row' do
+    let(:rows) { [] }
+
+    before do
+      allow(reader.category_reader.client).to receive(:get).and_call_original
+      allow(reader.category_reader.client).to receive(:get).with('categories/90509/children', anything).
+        and_return(categories: [{id: 8_476_099, childCount: 0}])
+
+      VCR.use_cassette 'read_models_from_category' do
+        reader.each_row { |row| rows << row }
+      end
+    end
+
+    it 'reads models of specified categories and their subcategories' do
+      expect(rows).to have(840).items
+      expect(rows.uniq).to have(840).items
+    end
+  end
+end

data/spec/fixtures/get_category_with_invalid_id.yml

@@ -0,0 +1,81 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://api.content.market.yandex.ru/v2/categories/999999999?geo_id=225
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Host:
+      - api.content.market.yandex.ru
+      Authorization:
+      - some_secret_token
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 01 Nov 2018 13:45:22 GMT
+      Content-Type:
+      - application/json;charset=utf-8
+      X-Request-Id:
+      - h9672x1101cf7fc849
+      X-Market-Req-Id:
+      - 1541079922406/bff948421ff4ffc38c089609f84b44cb
+      X-Region-Id:
+      - '225'
+      X-Ratelimit-Method-Limit:
+      - '100'
+      X-Ratelimit-Method-Remaining:
+      - '99'
+      X-Ratelimit-Method-Until:
+      - Thu, 01 Nov 2018 13:45:23 GMT
+      X-Ratelimit-Global-Limit:
+      - '2'
+      X-Ratelimit-Global-Remaining:
+      - '1'
+      X-Ratelimit-Global-Until:
+      - Thu, 01 Nov 2018 13:45:23 GMT
+      X-Ratelimit-Daily-Limit:
+      - '1000000000'
+      X-Ratelimit-Daily-Remaining:
+      - '999999508'
+      X-Ratelimit-Daily-Until:
+      - Fri, 02 Nov 2018 00:00:00 GMT
+      Content-Language:
+      - en-US
+      Cache-Control:
+      - no-store
+      Set-Cookie:
+      - i=nkVwTF0E65LH4sm4NpHIg5atL/I/faA0FmAPCndzPTcPoUFHMQZ6uan3HuO9B80ZasE1jpwvJexdt/1tRFSNHb9nnCk=;
+        Expires=Sun, 29-Oct-2028 13:45:22 GMT; Domain=.yandex.ru; Path=/; Secure;
+        HttpOnly
+      - uid=AAAihFvbA3KyYgE+BT9sAg==; path=/
+      Strict-Transport-Security:
+      - max-age=31536000
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        eyJzdGF0dXMiOiJFUlJPUiIsImNvbnRleHQiOnsicmVnaW9uIjp7ImlkIjoy
+        MjUsIm5hbWUiOiLQoNC+0YHRgdC40Y8iLCJ0eXBlIjoiQ09VTlRSWSIsImNo
+        aWxkQ291bnQiOjEwLCJjb3VudHJ5Ijp7ImlkIjoyMjUsIm5hbWUiOiLQoNC+
+        0YHRgdC40Y8iLCJ0eXBlIjoiQ09VTlRSWSIsImNoaWxkQ291bnQiOjEwfX0s
+        ImN1cnJlbmN5Ijp7ImlkIjoiUlVSIiwibmFtZSI6ItGA0YPQsS4ifSwiaWQi
+        OiIxNTQxMDc5OTIyNDA2L2JmZjk0ODQyMWZmNGZmYzM4YzA4OTYwOWY4NGI0
+        NGNiIiwidGltZSI6IjIwMTgtMTEtMDFUMTY6NDU6MjIuNDErMDM6MDAiLCJt
+        YXJrZXRVcmwiOiJodHRwczovL21hcmtldC55YW5kZXgucnU/cHA9MTAwMSZj
+        bGlkPTIzMjY2MDEmZGlzdHJfdHlwZT03In0sImVycm9ycyI6W3sibWVzc2Fn
+        ZSI6IkNhdGVnb3J5IDk5OTk5OTk5OSBub3QgZm91bmQifV19
+    http_version:
+  recorded_at: Thu, 01 Nov 2018 13:44:52 GMT
+recorded_with: VCR 4.0.0

data/spec/fixtures/get_root_categories.yml

@@ -0,0 +1,82 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://api.content.market.yandex.ru/v2/categories?count=30&geo_id=225
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Host:
+      - api.content.market.yandex.ru
+      Authorization:
+      - some_secret_token
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 24 Dec 2018 14:10:52 GMT
+      Content-Type:
+      - application/json;charset=utf-8
+      X-Request-Id:
+      - v4739x1224e089d5f0
+      X-Market-Req-Id:
+      - 1545660652508/11de5b809782bd8b8b4921f259e5bae2
+      X-Region-Id:
+      - '225'
+      X-Ratelimit-Method-Limit:
+      - '100'
+      X-Ratelimit-Method-Remaining:
+      - '99'
+      X-Ratelimit-Method-Until:
+      - Mon, 24 Dec 2018 14:10:53 GMT
+      X-Ratelimit-Global-Limit:
+      - '2'
+      X-Ratelimit-Global-Remaining:
+      - '1'
+      X-Ratelimit-Global-Until:
+      - Mon, 24 Dec 2018 14:10:53 GMT
+      X-Ratelimit-Daily-Limit:
+      - '1000000000'
+      X-Ratelimit-Daily-Remaining:
+      - '999997022'
+      X-Ratelimit-Daily-Until:
+      - Tue, 25 Dec 2018 00:00:00 GMT
+      Set-Cookie:
+      - i=9aI1w1reokMFM7TMMP4ppKWfu6YJf7jNGgKKpRqkF/l9nNpIMedEGcsqFZ1VsoOxgRMQPl/HH/1alGV5pMG3DgLJaJs=;
+        Expires=Thu, 21-Dec-2028 14:10:52 GMT; Domain=.yandex.ru; Path=/; Secure;
+        HttpOnly
+      - uid=AAAihFwg6OwlVgDIBCM0Ag==; path=/
+      Strict-Transport-Security:
+      - max-age=31536000
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: '{"status":"OK","context":{"region":{"id":225,"name":"Россия","type":"COUNTRY","childCount":10,"country":{"id":225,"name":"Россия","type":"COUNTRY","childCount":10}},"currency":{"id":"RUR","name":"руб."},"page":{"number":1,"count":18,"total":1},"id":"1545660652508/11de5b809782bd8b8b4921f259e5bae2","time":"2018-12-24T17:10:52.512+03:00","marketUrl":"https://market.yandex.ru?pp=1001&clid=2326601&distr_type=7"},"categories":[{"id":90402,"name":"Авто","fullName":"Товары
+        для авто- и мототехники","link":"https://market.yandex.ru/catalog--tovary-dlia-avto-i-mototekhniki/90402/list?hid=90402&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":12,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90509,"name":"Красота","fullName":"Товары
+        для красоты","link":"https://market.yandex.ru/catalog--tovary-dlia-krasoty/90509/list?hid=90509&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":3,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90666,"name":"Товары
+        для дома","fullName":"Товары для дома","link":"https://market.yandex.ru/catalog--tovary-dlia-doma/90666/list?hid=90666&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":9,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90719,"name":"Дача,
+        сад и огород","fullName":"Дача, сад и огород","link":"https://market.yandex.ru/catalog--dacha-sad-i-ogorod/90719/list?hid=90719&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":13,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90764,"name":"Детские
+        товары","fullName":"Детские товары","link":"https://market.yandex.ru/catalog--detskie-tovary/90764/list?hid=90764&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":6,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90801,"name":"Досуг
+        и развлечения","fullName":"Досуг и развлечения","link":"https://market.yandex.ru/catalog--dosug-i-razvlecheniia/90801/list?hid=90801&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":14,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":90813,"name":"Товары
+        для животных","fullName":"Товары для животных","link":"https://market.yandex.ru/catalog--tovary-dlia-zhivotnykh/90813/list?hid=90813&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":29,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91009,"name":"Компьютерная
+        техника","fullName":"Компьютерная техника","link":"https://market.yandex.ru/catalog--kompiuternaia-tekhnika/91009/list?hid=91009&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":12,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91307,"name":"Продукты","fullName":"Продукты,
+        напитки","link":"https://market.yandex.ru/catalog--produkty-napitki/91307/list?hid=91307&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":14,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91512,"name":"Спорт
+        и отдых","fullName":"Товары для спорта и отдыха","link":"https://market.yandex.ru/catalog--tovary-dlia-sporta-i-otdykha/91512/list?hid=91512&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":17,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91597,"name":"Строительство
+        и ремонт","fullName":"Товары для строительства и ремонта","link":"https://market.yandex.ru/catalog--tovary-dlia-stroitelstva-i-remonta/91597/list?hid=91597&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":7,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":198119,"name":"Электроника","fullName":"Электроника","link":"https://market.yandex.ru/catalog--elektronika/198119/list?hid=198119&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":7,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":198118,"name":"Бытовая
+        техника","fullName":"Бытовая техника","link":"https://market.yandex.ru/catalog--bytovaia-tekhnika/198118/list?hid=198118&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":6,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91735,"name":"Услуги","fullName":"Услуги","link":"https://market.yandex.ru/catalog--uslugi/91735/list?hid=91735&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":11,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":91763,"name":"Оборудование","fullName":"Оборудование","link":"https://market.yandex.ru/catalog--oborudovanie/91763/list?hid=91763&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":16,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":15807107,"name":"Услуги
+        по подписке","fullName":"Услуги по подписке","link":"https://market.yandex.ru/catalog--uslugi-po-podpiske/15807107/list?hid=15807107&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":0,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":8475840,"name":"Здоровье","fullName":"Товары
+        для здоровья","link":"https://market.yandex.ru/catalog--tovary-dlia-zdorovia/8475840/list?hid=8475840&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":15,"advertisingModel":"HYBRID","viewType":"LIST"},{"id":7877999,"name":"Одежда,
+        обувь и аксессуары","fullName":"Одежда, обувь и аксессуары","link":"https://market.yandex.ru/catalog--odezhda-obuv-i-aksessuary/7877999/list?hid=7877999&onstock=1&pp=1001&clid=2326601&distr_type=7","childCount":5,"advertisingModel":"HYBRID","viewType":"LIST"}]}'
+    http_version:
+  recorded_at: Mon, 24 Dec 2018 14:10:02 GMT
+recorded_with: VCR 4.0.0