apress-yandex_market 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b62a9a0c7db6767fe9559e3e07cfefd0b1bed4f
+  data.tar.gz: 712a1629589d5f6db999c62fb6f3b547e49cd705
+SHA512:
+  metadata.gz: 4a0840fceb8743f97872f44240717414b55cf0dc3632dd24c6bc22511375cb98bf267e97cff3bbf2b38157c7feceb515d5cdb8aaad1f966d181b1a7a6d9fa19d
+  data.tar.gz: 4b605819f184624087a2ff58bf156560ebf84af744df6977250d0589daf5d4c7ab049f9b4c8b9d36445ced72749b9897b7700b290d0c88dfb96ba0c31302473d

data/.drone.yml

@@ -0,0 +1,30 @@
+build:
+  test:
+    image: abakpress/dind-testing
+    pull: true
+    privileged: true
+    volumes:
+      - /home/data/drone/images:/images
+      - /home/data/drone/gems:/bundle
+    environment:
+      - COMPOSE_FILE_EXT=drone
+      - RUBY_IMAGE_TAG=2.2-latest
+    commands:
+      - prepare-build
+
+      - fetch-images
+        --image abakpress/ruby-app:$RUBY_IMAGE_TAG
+
+      - dip provision
+      - dip rspec
+
+  release:
+    image: abakpress/gem-publication:latest
+    pull: true
+    when:
+      event: push
+      branch: master
+    volumes:
+      - /home/data/drone/rubygems:/root/.gem
+    commands:
+      - release-gem --public

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/Gemfile.lock
+/coverage/
+/doc/
+/pkg/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+/.rvmrc
+/.idea/
+gemfiles/

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--order random

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# v0.1.0
+
+* 2018-12-24 [b5adb2b](../../commit/b5adb2b) - __(Mikhail Nelaev)__ fix: получаем все корневые категории 
+https://jira.railsc.ru/browse/GOODS-1525
+
+* 2018-11-12 [e660187](../../commit/e660187) - __(Mikhail Nelaev)__ feature: обновление по api Яндекс.Маркета 
+https://jira.railsc.ru/browse/GOODS-1525
+
+* 2018-11-08 [113a81b](../../commit/113a81b) - __(Mamedaliev Kirill)__ Initial commit 

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,3 @@
+# Apress::YandexMarket
+
+Библиотека для работы с Яндекс.Маркет

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'rspec/core/rake_task'
+
+# setup `spec` task
+RSpec::Core::RakeTask.new(:spec)

data/apress-yandexmarket.gemspec

@@ -0,0 +1,36 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include? lib
+
+require 'apress/yandex_market/version'
+
+Gem::Specification.new do |gem|
+  gem.name         = 'apress-yandex_market'
+  gem.version      = Apress::YandexMarket::VERSION
+  gem.authors      = ['Mikhail Nelaev']
+  gem.email        = %w(spyderdfx@gmail.com)
+  gem.summary      = 'Tools for synchronization with Yandex.Market'
+  gem.homepage     = 'https://github.com/abak-press/apress-yandex_market'
+
+  gem.files        = `git ls-files -z`.split("\x0")
+  gem.test_files   = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_path = "lib"
+
+  gem.add_runtime_dependency 'oj'
+  gem.add_runtime_dependency 'facets'
+
+  gem.add_development_dependency 'bundler', '~> 1.6'
+  gem.add_development_dependency 'rake', '< 11.0'  # https://github.com/lsegal/yard/issues/947
+
+  # test
+  gem.add_development_dependency 'rspec', '>= 3.5'
+  gem.add_development_dependency 'rspec-its'
+  gem.add_development_dependency 'rspec-collection_matchers'
+  gem.add_development_dependency 'webmock', '< 2.3'
+  gem.add_development_dependency 'vcr'
+
+  # test coverage tools
+  gem.add_development_dependency 'simplecov', '~> 0.10.0'
+
+  # debug
+  gem.add_development_dependency 'pry-byebug'
+end

data/dip.yml

@@ -0,0 +1,41 @@
+version: '2'
+
+environment:
+  DOCKER_RUBY_VERSION: 2.2
+  RUBY_IMAGE_TAG: 2.2-latest
+  COMPOSE_FILE_EXT: development
+  RAILS_ENV: test
+
+compose:
+  files:
+    - docker-compose.yml
+    - docker-compose.${COMPOSE_FILE_EXT}.yml
+
+interaction:
+  sh:
+    service: app
+
+  irb:
+    service: app
+    command: irb
+
+  bundle:
+    service: app
+    command: bundle
+
+  rake:
+    service: app
+    command: bundle exec rake
+
+  rspec:
+    service: app
+    command: bundle exec rspec
+
+  clean:
+    service: app
+    command: rm -f Gemfile.lock gemfiles/*.gemfile.*
+
+provision:
+  - docker volume create --name bundler_data
+  - dip clean
+  - dip bundle install

data/docker-compose.development.yml

@@ -0,0 +1,13 @@
+version: '2'
+
+services:
+  app:
+    volumes:
+      - .:/app
+      - ../:/localgems
+      - bundler-data:/bundle
+
+volumes:
+  bundler-data:
+    external:
+      name: bundler_data

data/docker-compose.drone.yml

@@ -0,0 +1,7 @@
+version: '2'
+
+services:
+  app:
+    volumes:
+      - .:/app
+      - /bundle:/bundle

data/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '2'
+
+services:
+  app:
+    image: abakpress/ruby-app:$RUBY_IMAGE_TAG
+    environment:
+      - BUNDLE_PATH=/bundle/$DOCKER_RUBY_VERSION
+      - BUNDLE_CONFIG=/app/.bundle/config
+    command: bash

data/lib/apress/yandex_market/api/client.rb

@@ -0,0 +1,79 @@
+require 'net/http'
+require 'openssl'
+require 'uri'
+require 'oj'
+
+module Apress
+  module YandexMarket
+    module Api
+      # API клиент Яндекс.Маркета
+      class Client
+        TIMEOUT = 60
+        HOST = 'api.content.market.yandex.ru'.freeze
+        ACCEPT = '*/*'.freeze
+        VERSION = 'v2'.freeze
+
+        # Public: инициализация клиента
+        #
+        # auth_token - String, токен для доступа к API
+        #
+        # Returns an instance of Api::Client
+        def initialize(auth_token)
+          @auth_token = auth_token
+        end
+
+        # Public: выполняет GET-запрос и парсит ответ в формате JSON
+        #
+        # path   - String, ресурс API
+        # params - Hash, дополнительные get-параметры запроса (default: {})
+        #
+        # Returns Hash
+        def get(path, params = {})
+          uri = api_uri(path)
+          uri.query = URI.encode_www_form(params) unless params.empty?
+
+          req = Net::HTTP::Get.new(uri)
+          req['Host'] = HOST
+          req['Accept'] = ACCEPT
+          req['Authorization'] = @auth_token
+
+          res = Net::HTTP.start(uri.hostname, uri.port, http_options) do |http|
+            http.request(req)
+          end
+
+          if res['Content-Type'] && res['Content-Type'].start_with?('application/json')
+            parse_response(res)
+          else
+            raise Api::Error.new(res.msg, res.code)
+          end
+        end
+
+        private
+
+        def api_uri(path)
+          URI "https://#{HOST}/#{VERSION}/#{path}"
+        end
+
+        def parse_response(res)
+          Oj.load(res.body, symbol_keys: true, mode: :compat).tap do |data|
+            if data[:status] == 'ERROR'
+              err = data[:errors].first
+
+              raise Api::Error.new(err[:message], res.code)
+            end
+
+            raise Api::Error.new(res.msg, res.code) unless res.is_a? Net::HTTPOK
+          end
+        end
+
+        def http_options
+          {
+            open_timeout: TIMEOUT,
+            read_timeout: TIMEOUT,
+            use_ssl: true
+          }
+        end
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/api/error.rb

@@ -0,0 +1,19 @@
+module Apress
+  module YandexMarket
+    module Api
+      class Error < StandardError
+        attr_reader :code
+
+        def initialize(msg, code = nil)
+          @code = code.to_i
+
+          message = code ? "#{code} - #{msg}" : msg
+
+          raise PageError.new(message) if msg.start_with? Api::PageError::MSG
+
+          super message.force_encoding('UTF-8')
+        end
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/api/page_error.rb

@@ -0,0 +1,9 @@
+module Apress
+  module YandexMarket
+    module Api
+      class PageError < StandardError
+        MSG = "Parameter 'page' has invalid value. Parameter does not fit range constraint".freeze
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/api.rb

@@ -0,0 +1,9 @@
+module Apress
+  module YandexMarket
+    module Api
+      autoload :Client, 'apress/yandex_market/api/client'
+      autoload :Error, 'apress/yandex_market/api/error'
+      autoload :PageError, 'apress/yandex_market/api/page_error'
+    end
+  end
+end

data/lib/apress/yandex_market/presenters/base.rb

@@ -0,0 +1,49 @@
+module Apress
+  module YandexMarket
+    module Presenters
+      class Base
+        class << self
+          attr_accessor :attributes
+
+          attributes = []
+        end
+
+        def initialize
+          @counter = 0
+        end
+
+        def expose(row)
+          record = {
+            __line__: @counter += 1,
+            __column__: 1
+          }
+
+          record.merge! filter(row)
+
+          record
+        end
+
+        private
+
+        def filter(row, attrs = self.class.attributes)
+          attrs.each_with_object({}) do |key, result|
+            if key.is_a? Hash
+              key.each do |hash_key, hash_attrs|
+                next unless row.key? hash_key
+
+                result[hash_key] =
+                  if row[hash_key].is_a? Array
+                    row[hash_key].map { |item| filter(item, hash_attrs) }
+                  else
+                    filter(row[hash_key], hash_attrs)
+                  end
+              end
+            elsif row.key? key
+              result[key] = row[key]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/presenters/model.rb

@@ -0,0 +1,19 @@
+module Apress
+  module YandexMarket
+    module Presenters
+      class Model < Base
+        self.attributes = [
+          :id,
+          :name,
+          :link,
+          :description,
+          photo: [:url].freeze,
+          photos: [:url].freeze,
+          price: [:min].freeze,
+          vendor: [:name].freeze,
+          specification: [features: [:name, :value].freeze].freeze
+        ].freeze
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/presenters.rb

@@ -0,0 +1,8 @@
+module Apress
+  module YandexMarket
+    module Presenters
+      autoload :Base, 'apress/yandex_market/presenters/base'
+      autoload :Model, 'apress/yandex_market/presenters/model'
+    end
+  end
+end

data/lib/apress/yandex_market/readers/base.rb

@@ -0,0 +1,54 @@
+require 'apress/yandex_market/api'
+
+module Apress
+  module YandexMarket
+    module Readers
+      # Базовый класс ридеров
+      class Base
+        DEFAULT_REGION_ID = 225 # Россия
+        PAGE_SIZE = 30 # максимум сущностей на одной странице в API Яндекс.Маркета
+
+        SLEEP_TIME = 0.5 # no more than 2 rps
+
+        RETRY_ATTEMPTS_SLEEP_TIME = [60, 60, 30, 15, 1].freeze
+        RETRY_ATTEMPTS = RETRY_ATTEMPTS_SLEEP_TIME.size
+        RETRY_CODES = [401, 403, 404, 500, 502, 503, 504].freeze
+
+        attr_reader :client
+
+        class << self
+          def allowed_options
+            %i(token region_id)
+          end
+        end
+
+        # Public: инициализация ридера
+        #
+        # options - Hash, параметры ридера
+        #   :region_id - идентификатор региона (необязательный)
+        #   :token     - токен для доступа к API Яндекс.Маркета
+        #
+        # Returns an instance of Readers::Base
+        def initialize(options)
+          @region_id = options.fetch(:region_id, DEFAULT_REGION_ID)
+          @client = Api::Client.new(options.fetch(:token))
+        end
+
+        private
+
+        def with_rescue_temporary_errors(attempts = RETRY_ATTEMPTS)
+          yield
+        rescue Api::Error, Timeout::Error => err
+          raise if err.is_a?(Api::Error) && !RETRY_CODES.include?(err.code)
+
+          if (attempts -= 1) > 0
+            sleep RETRY_ATTEMPTS_SLEEP_TIME[attempts]
+            retry
+          else
+            raise
+          end
+        end
+      end
+    end
+  end
+end

data/lib/apress/yandex_market/readers/category.rb

@@ -0,0 +1,117 @@
+require 'apress/yandex_market/readers/base'
+require 'facets/string/squish'
+
+module Apress
+  module YandexMarket
+    module Readers
+      # Ридер категорий. Читает из API Яндекс.Маркета переданные категории и их подкатегории
+      #
+      # Examples
+      #
+      #   reader = Apress::YandexMarket::Readers::Category.new(region_id: 225,
+      #                                                        token: 'qwerty',
+      #                                                        categories: 'Авто; Дом и дача')
+      #   # => #<Apress::YandexMarket::Readers::Category ...>
+      #
+      #   reader.each_row |category|
+      #     puts category
+      #   end
+      #   # => {
+      #          :id=>90402,
+      #          :name=>"Авто",
+      #          :fullName=>"Товары для авто- и мототехники",
+      #          :link=>
+      #            "https://market.yandex.ru/catalog/90402/list?hid=90402&onstock=1&pp=1001&clid=2326601&distr_type=7",
+      #          :childCount=>12,
+      #          :advertisingModel=>"HYBRID",
+      #          :viewType=>"LIST"
+      #        }
+      #        {
+      #          :id=>90403,
+      #          ...
+      #        }
+      class Category < Base
+        FIELDS = %w(PARENT).join(',').freeze
+
+        class << self
+          def allowed_options
+            super + %i(categories)
+          end
+        end
+
+        # Public: инициализация ридера
+        #
+        # options - Hash, параметры ридера
+        #   :region_id  - идентификатор региона (необязательный)
+        #   :token      - токен для доступа к API Яндекс.Маркета
+        #   :categories - список категорий для загрузки моделей Яндекс.Маркета (разделитель - ';')
+        #
+        # Returns an instance of Readers::Category
+        def initialize(options)
+          super
+          @categories = options.fetch(:categories).split(';').map(&:squish)
+        end
+
+        # Public: читаем категории из API и для каждой выполняем блок
+        #
+        # Returns nothing
+        def each_row
+          root_categories.each do |root_category|
+            yield root_category
+
+            get_children_categories(root_category[:id]) do |category|
+              yield category
+            end
+          end
+        end
+
+        private
+
+        def get_children_categories(parent_id, &block)
+          page = 1
+
+          loop do
+            sleep(SLEEP_TIME)
+            categories =
+              get_categories("categories/#{parent_id}/children", page)
+
+            page += 1
+
+            categories.each do |category|
+              yield category
+
+              next if category[:childCount].zero?
+
+              get_children_categories(category[:id], &block)
+            end
+
+            break if categories.empty? || categories.count < PAGE_SIZE
+          end
+        end
+
+        def root_categories(page = 1)
+          categories = get_categories('categories', page)
+          categories += root_categories(page + 1) unless categories.size < PAGE_SIZE
+
+          categories.select! { |category| @categories.include?(category.fetch(:name)) }
+        end
+
+        def get_categories(resource, page)
+          with_rescue_temporary_errors do
+            client.
+              get(
+                resource,
+                geo_id: @region_id,
+                sort: 'BY_NAME'.freeze,
+                count: PAGE_SIZE,
+                page: page
+              ).
+              fetch(:categories)
+          end
+        rescue Api::PageError
+          []
+        end
+      end
+    end
+  end
+end