base_scraper_service 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 69eb49ba30aa10adba230399397bc900df7f46b10da8c5ef7e02d00743fd7407
+  data.tar.gz: a7534672884d78323aa41838e1048c1cfff7a59d19a7f633034eaf9e8dc680d0
+SHA512:
+  metadata.gz: b8957504e31a3a5519538bd7b5fecdb7837f502b5cdccea51c59d3c585bb26f5dc7642449808e3913d361ee778af74e06708dc54b5ed771cb8839434f1e7834c
+  data.tar.gz: 6227de3f572c770425145ef074dbbdbec0cebf8cd0c7f95ba4cc1d00a0ef24e60534e3537d5ae305d1a6af41688304b55fea00b4bd78b6a3b0e951ee5f28dea4

data/lib/base_scraper_service/agent_object.rb

@@ -0,0 +1,28 @@
+require_relative 'user_agent'
+require "mechanize"
+require "nokogiri"
+
+module BaseScraper
+  class Service
+    def agent_object
+      agent = Mechanize.new
+      agent.read_timeout = 5
+      agent.open_timeout = 8
+      agent.keep_alive = false
+      agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      agent.user_agent = UserAgent.random
+      agent.idle_timeout = 5
+      agent.pluggable_parser.default = Mechanize::Page
+
+      # Prevent from <Mechanize::Error: unsupported content-encoding: UTF-8>
+      content_encoding_hooks_func = lambda do |a, uri, resp, body_io|
+        if resp['Content-Encoding'].to_s == 'UTF-8'
+          resp['Content-Encoding'] = 'gzip'
+        end
+      end
+      agent.content_encoding_hooks << content_encoding_hooks_func
+
+      agent
+    end
+  end
+end

data/lib/base_scraper_service/base.rb

@@ -0,0 +1,94 @@
+require 'sinatra'
+require 'dotenv'
+require "./#{ENV['SERVICE_NAME']}_offers_service"
+
+configure { set :server, :puma }
+
+class Base < Sinatra::Base
+
+  get "/#{ENV['SERVICE_NAME']}/isbn_offers" do
+    isbn = params[:isbn]
+    uuid = SecureRandom.uuid
+
+    @res = {
+      isbn: isbn,
+      uuid: uuid,
+      item_condition: params[:item_condition],
+      ship_type: params[:ship_type]
+    }
+
+    if valid_params?
+      service_response = Object.const_get(get_service_name).new(isbn, uuid).scrape_offers
+      offers = service_response[:offers]
+
+      offers = filter_offers(offers)
+      offers.sort_by! { |offer| offer[:total_price] }
+      offers_count = offers.count
+
+      @res[:status] = service_response[:status]
+      @res[:short_message] = service_response[:short_message]
+      @res[:long_message] = service_response[:long_message]
+      if @res[:status] == 200
+        @res[:offers_count] = offers.count
+        @res[:data] = { offers: offers }
+      end
+    else
+      @res[:status] = 400
+      @res[:short_message] = 'error'
+    end
+
+    halt @res[:status], @res.to_json
+  end
+
+  not_found do
+    send_file( 'public/404.html', { status: 404 })
+  end
+
+  def get_service_name
+    ENV['SERVICE_NAME'].capitalize + 'OffersService'
+  end
+
+  def send_sinatra_file(path)
+    file_path = File.join(File.dirname(__FILE__), 'public',  path)
+    file_path = File.join(file_path, 'index.html') unless file_path =~ /\.[a-z]+$/i and !File.directory?(file_path)
+    File.exist?(file_path) ? send_file(file_path) : not_found
+  end
+
+  def filter_offers(offers)
+    offers = offers.select{ |offer| offer[:Item_Condition].downcase ==  params[:item_condition].downcase} if params[:item_condition].present?
+    offers = offers.select{ |offer| offer[:ship_type].downcase ==  params[:ship_type].downcase} if params[:ship_type].present?
+
+    offers
+  end
+
+  def constantize(lower_case_string)
+    "#{lower_case_string.split('_').collect(&:capitalize).join + 'OffersService'}".constantize
+  end
+
+  def valid_params?
+    valid_isbn?(params[:isbn]) &&
+    valid_book_condition?(params[:item_condition].to_s) &&
+    valid_ship_type?(params[:ship_type].to_s)
+  end
+
+  def valid_isbn?(isbn)
+    return true if StdNum::ISBN.valid?(isbn) && [10, 13].include?(isbn.size)
+
+    @res[:long_message] = 'invalid isbn'
+    false
+  end
+
+  def valid_book_condition?(item_condition)
+    return true if ['', 'used', 'new'].include?(item_condition.downcase)
+
+    @res[:long_message] = 'invalid item condition'
+    false
+  end
+
+  def valid_ship_type?(ship_type)
+    return true if ['', 'standard', 'expedited'].include?(ship_type.downcase)
+
+    @res[:long_message] = 'invalid ship type'
+    false
+  end
+end

data/lib/base_scraper_service/base_scraper.rb

@@ -0,0 +1,90 @@
+require 'mechanize'
+require 'nokogiri'
+require 'thread'
+require 'fileutils'
+require 'logger'
+require "dotenv"
+require 'csv'
+require 'date'
+require 'active_support/time'
+require 'library_stdnums'
+require 'base_scraper_service'
+require_relative 'user_agent'
+require_relative 'agent_object'
+
+module BaseScraper
+  class Service
+    def send_request(agent, uri, params = nil, headers = {}, _optoins = {})
+      tries = 0
+      max_tries = 5
+      page = nil
+
+      refresh_agent(agent)
+      change_proxy(agent)
+      begin
+        page = params.nil? ? agent.get(uri, [], nil, headers) : agent.post(uri, params, headers)
+
+      rescue Mechanize::ResponseCodeError => e
+        change_proxy(agent)
+
+        tries += 1
+
+        agent.set_proxy(ENV["LPM_SERVER_URL"], ENV["LPM_SERVER_PORT"]) if tries >= 4
+
+        retry if tries < max_tries
+      rescue Exception => e
+        refresh_agent(agent)
+        change_proxy(agent)
+
+        tries += 1
+
+        agent.set_proxy(ENV["LPM_SERVER_URL"], ENV["LPM_SERVER_PORT"]) if tries >= 4
+
+        retry if tries < max_tries
+        puts "Message: #{e.message} from #{uri}, proxy: #{agent.proxy_addr}" if logs_enable?
+      end
+
+      page
+    end
+
+    def format_price(price)
+      return unless price.present?
+      price.to_s.strip.gsub("$", "").strip.to_f
+    end
+
+    def format_isbn(isbn)
+      StdNum::ISBN.convert_to_13(isbn)
+    end
+
+    def with_exception(long_message: "", status: 500, short_message: "exception")
+      return { offers: [], short_message: short_message, long_message: long_message, status: status }
+    end
+
+    def logs_enable?
+      ENV["ENABLE_LOGS"] == "true"
+    end
+
+    private
+
+    def refresh_agent(agent)
+      agent.cookie_jar.clear!
+      agent.history.clear
+      agent.user_agent = UserAgent.random
+    end
+
+    def get_proxy_index
+      (0..(@proxy_array.count - 1)).to_a.sample
+    end
+
+    def change_proxy(agent)
+      if @proxy_array.count > 0
+        cur_proxy_index = get_proxy_index
+        if @proxy_array[cur_proxy_index].present?
+          agent.set_proxy(@proxy_array[cur_proxy_index][0], @proxy_array[cur_proxy_index][1])
+        end
+      else
+        agent.set_proxy(ENV['LPM_SERVER_URL'], ENV['LPM_SERVER_PORT'])
+      end
+    end
+  end
+end

data/lib/base_scraper_service/location_service.rb

@@ -0,0 +1,348 @@
+module BaseScraper
+  class Service
+    class LocationService
+      def initialize(address)
+        @address = address.to_s.gsub('.', '')
+      end
+
+      def extract_state
+        # 1- return state abbreviation if matched with state abbreviations
+        # 2- return state abbreviation if matched with state name
+        # 3- otherwise return empty string
+        state = us_states_hash.keys.select { |state_abbreviation| @address.upcase.include?(state_abbreviation) }.first
+        return state if state.present? && @address.length != 3
+        state = us_states_hash.select { |_, state| @address.include?(state) }.first&.first
+
+        return state.to_s
+      end
+
+      def extract_country
+        # 1- return country abbreviation if matched with country abbreviations
+        # 2- return country abbreviation if matched with country name
+        # 3- return USA if address contains a USA state
+        # 4- otherwise return empty string
+        country = countries_hash.keys.select { |country_abbreviation| @address.include?(country_abbreviation) }.first
+        puts country, @address
+        return country if country.present?
+        country = countries_hash.select { |_, country_name| @address.include?(country_name) }.first&.first
+        return country if country.present?
+
+        return 'USA' if extract_state.present?
+
+        return country.to_s
+      end
+
+      private
+
+      def us_states_hash
+        {
+          'AL' => 'Alabama',
+          'AK' => 'Alaska',
+          'AZ' => 'Arizona',
+          'AR' => 'Arkansas',
+          'CA' => 'California',
+          'CO' => 'Colorado',
+          'CT' => 'Connecticut',
+          'DE' => 'Delaware',
+          'DC' => 'District Of Columbia',
+          'FL' => 'Florida',
+          'GA' => 'Georgia',
+          'HI' => 'Hawaii',
+          'ID' => 'Idaho',
+          'IL' => 'Illinois',
+          'IN' => 'Indiana',
+          'IA' => 'Iowa',
+          'KS' => 'Kansas',
+          'KY' => 'Kentucky',
+          'LA' => 'Louisiana',
+          'ME' => 'Maine',
+          'MD' => 'Maryland',
+          'MA' => 'Massachusetts',
+          'MI' => 'Michigan',
+          'MN' => 'Minnesota',
+          'MS' => 'Mississippi',
+          'MO' => 'Missouri',
+          'MT' => 'Montana',
+          'NE' => 'Nebraska',
+          'NV' => 'Nevada',
+          'NH' => 'New Hampshire',
+          'NJ' => 'New Jersey',
+          'NM' => 'New Mexico',
+          'NY' => 'New York',
+          'NC' => 'North Carolina',
+          'ND' => 'North Dakota',
+          'OH' => 'Ohio',
+          'OK' => 'Oklahoma',
+          'OR' => 'Oregon',
+          'PA' => 'Pennsylvania',
+          'RI' => 'Rhode Island',
+          'SC' => 'South Carolina',
+          'SD' => 'South Dakota',
+          'TN' => 'Tennessee',
+          'TX' => 'Texas',
+          'UT' => 'Utah',
+          'VT' => 'Vermont',
+          'VA' => 'Virginia',
+          'WA' => 'Washington',
+          'WV' => 'West Virginia',
+          'WI' => 'Wisconsin',
+          'WY' => 'Wyoming'
+        }
+      end
+
+      def countries_hash
+        {
+          'AFG' => 'Afghanistan',
+          'ALB' => 'Albania',
+          'DZA' => 'Algeria',
+          'ASM' => 'American Samoa',
+          'AND' => 'Andorra',
+          'AGO' => 'Angola',
+          'AIA' => 'Anguilla',
+          'ATA' => 'Antarctica',
+          'ATG' => 'Antigua and Barbuda',
+          'ARG' => 'Argentina',
+          'ARM' => 'Armenia',
+          'ABW' => 'Aruba',
+          'AUS' => 'Australia',
+          'AUT' => 'Austria',
+          'AZE' => 'Azerbaijan',
+          'BHS' => 'Bahamas',
+          'BHR' => 'Bahrain',
+          'BGD' => 'Bangladesh',
+          'BRB' => 'Barbados',
+          'BLR' => 'Belarus',
+          'BEL' => 'Belgium',
+          'BLZ' => 'Belize',
+          'BEN' => 'Benin',
+          'BMU' => 'Bermuda',
+          'BTN' => 'Bhutan',
+          'BOL' => 'Bolivia',
+          'BES' => 'Bonaire, Sint Eustatius and Saba',
+          'BIH' => 'Bosnia and Herzegovina',
+          'BWA' => 'Botswana',
+          'BVT' => 'Bouvet Island',
+          'BRA' => 'Brazil',
+          'IOT' => 'British Indian Ocean Territory',
+          'BRN' => 'Brunei Darussalam',
+          'BGR' => 'Bulgaria',
+          'BFA' => 'Burkina Faso',
+          'BDI' => 'Burundi',
+          'CPV' => 'Cabo Verde',
+          'KHM' => 'Cambodia',
+          'CMR' => 'Cameroon',
+          'CAN' => 'Canada',
+          'CYM' => 'Cayman Islands',
+          'CAF' => 'Central African Republic',
+          'TCD' => 'Chad',
+          'CHL' => 'Chile',
+          'CHN' => 'China',
+          'CXR' => 'Christmas Island',
+          'CCK' => 'Cocos (Keeling) Islands',
+          'COL' => 'Colombia',
+          'COM' => 'Comoros',
+          'COD' => 'Congo',
+          'COG' => 'Congo',
+          'COK' => 'Cook Islands',
+          'CRI' => 'Costa Rica',
+          'HRV' => 'Croatia',
+          'CUB' => 'Cuba',
+          'CUW' => 'Curaçao',
+          'CYP' => 'Cyprus',
+          'CZE' => 'Czechia',
+          'CIV' => "Côte d'Ivoire",
+          'DNK' => 'Denmark',
+          'DJI' => 'Djibouti',
+          'DMA' => 'Dominica',
+          'DOM' => 'Dominican Republic',
+          'ECU' => 'Ecuador',
+          'EGY' => 'Egypt',
+          'SLV' => 'El Salvador',
+          'GNQ' => 'Equatorial Guinea',
+          'ERI' => 'Eritrea',
+          'EST' => 'Estonia',
+          'SWZ' => 'Eswatini',
+          'ETH' => 'Ethiopia',
+          'FLK' => 'Falkland Islands',
+          'FRO' => 'Faroe Islands',
+          'FJI' => 'Fiji',
+          'FIN' => 'Finland',
+          'FRA' => 'France',
+          'GUF' => 'French Guiana',
+          'PYF' => 'French Polynesia',
+          'ATF' => 'French Southern Territories',
+          'GAB' => 'Gabon',
+          'GMB' => 'Gambia',
+          'GEO' => 'Georgia',
+          'DEU' => 'Germany',
+          'GHA' => 'Ghana',
+          'GIB' => 'Gibraltar',
+          'GRC' => 'Greece',
+          'GRL' => 'Greenland',
+          'GRD' => 'Grenada',
+          'GLP' => 'Guadeloupe',
+          'GUM' => 'Guam',
+          'GTM' => 'Guatemala',
+          'GGY' => 'Guernsey',
+          'GIN' => 'Guinea',
+          'GNB' => 'Guinea-Bissau',
+          'GUY' => 'Guyana',
+          'HTI' => 'Haiti',
+          'HMD' => 'Heard Island and McDonald Islands',
+          'VAT' => 'Holy See',
+          'HND' => 'Honduras',
+          'HKG' => 'Hong Kong',
+          'HUN' => 'Hungary',
+          'ISL' => 'Iceland',
+          'IND' => 'India',
+          'IDN' => 'Indonesia',
+          'IRN' => 'Iran',
+          'IRQ' => 'Iraq',
+          'IRL' => 'Ireland',
+          'IMN' => 'Isle of Man',
+          'ISR' => 'Israel',
+          'ITA' => 'Italy',
+          'JAM' => 'Jamaica',
+          'JPN' => 'Japan',
+          'JEY' => 'Jersey',
+          'JOR' => 'Jordan',
+          'KAZ' => 'Kazakhstan',
+          'KEN' => 'Kenya',
+          'KIR' => 'Kiribati',
+          'PRK' => "Korea",
+          'KOR' => 'Korea',
+          'KWT' => 'Kuwait',
+          'KGZ' => 'Kyrgyzstan',
+          'LAO' => "Lao People's Democratic Republic",
+          'LVA' => 'Latvia',
+          'LBN' => 'Lebanon',
+          'LSO' => 'Lesotho',
+          'LBR' => 'Liberia',
+          'LBY' => 'Libya',
+          'LIE' => 'Liechtenstein',
+          'LTU' => 'Lithuania',
+          'LUX' => 'Luxembourg',
+          'MAC' => 'Macao',
+          'MDG' => 'Madagascar',
+          'MWI' => 'Malawi',
+          'MYS' => 'Malaysia',
+          'MDV' => 'Maldives',
+          'MLI' => 'Mali',
+          'MLT' => 'Malta',
+          'MHL' => 'Marshall Islands',
+          'MTQ' => 'Martinique',
+          'MRT' => 'Mauritania',
+          'MUS' => 'Mauritius',
+          'MYT' => 'Mayotte',
+          'MEX' => 'Mexico',
+          'FSM' => 'Micronesia',
+          'MDA' => 'Moldova',
+          'MCO' => 'Monaco',
+          'MNG' => 'Mongolia',
+          'MNE' => 'Montenegro',
+          'MSR' => 'Montserrat',
+          'MAR' => 'Morocco',
+          'MOZ' => 'Mozambique',
+          'MMR' => 'Myanmar',
+          'NAM' => 'Namibia',
+          'NRU' => 'Nauru',
+          'NPL' => 'Nepal',
+          'NLD' => 'Netherlands',
+          'NCL' => 'New Caledonia',
+          'NZL' => 'New Zealand',
+          'NIC' => 'Nicaragua',
+          'NER' => 'Niger',
+          'NGA' => 'Nigeria',
+          'NIU' => 'Niue',
+          'NFK' => 'Norfolk Island',
+          'MNP' => 'Northern Mariana Islands',
+          'NOR' => 'Norway',
+          'OMN' => 'Oman',
+          'PAK' => 'Pakistan',
+          'PLW' => 'Palau',
+          'PSE' => 'Palestine, State of',
+          'PAN' => 'Panama',
+          'PNG' => 'Papua New Guinea',
+          'PRY' => 'Paraguay',
+          'PER' => 'Peru',
+          'PHL' => 'Philippines',
+          'PCN' => 'Pitcairn',
+          'POL' => 'Poland',
+          'PRT' => 'Portugal',
+          'PRI' => 'Puerto Rico',
+          'QAT' => 'Qatar',
+          'MKD' => 'Republic of North Macedonia',
+          'ROU' => 'Romania',
+          'RUS' => 'Russian Federation',
+          'RWA' => 'Rwanda',
+          'REU' => 'Réunion',
+          'BLM' => 'Saint Barthélemy',
+          'SHN' => 'Saint Helena, Ascension and Tristan da Cunha',
+          'KNA' => 'Saint Kitts and Nevis',
+          'LCA' => 'Saint Lucia',
+          'MAF' => 'Saint Martin',
+          'SPM' => 'Saint Pierre and Miquelon',
+          'VCT' => 'Saint Vincent and Grenadines',
+          'WSM' => 'Samoa',
+          'SMR' => 'San Marino',
+          'STP' => 'Sao Tome and Principe',
+          'SAU' => 'Saudi Arabia',
+          'SEN' => 'Senegal',
+          'SRB' => 'Serbia',
+          'SYC' => 'Seychelles',
+          'SLE' => 'Sierra Leone',
+          'SGP' => 'Singapore',
+          'SXM' => 'Sint Maarten',
+          'SVK' => 'Slovakia',
+          'SVN' => 'Slovenia',
+          'SLB' => 'Solomon Islands',
+          'SOM' => 'Somalia',
+          'ZAF' => 'South Africa',
+          'SGS' => 'South Georgia and South Sandwich Islands',
+          'SSD' => 'South Sudan',
+          'ESP' => 'Spain',
+          'LKA' => 'Sri Lanka',
+          'SDN' => 'Sudan',
+          'SUR' => 'Suriname',
+          'SJM' => 'Svalbard and Jan Mayen',
+          'SWE' => 'Sweden',
+          'CHE' => 'Switzerland',
+          'SYR' => 'Syrian Arab Republic',
+          'TWN' => 'Taiwan',
+          'TJK' => 'Tajikistan',
+          'TZA' => 'Tanzania, United Republic of',
+          'THA' => 'Thailand',
+          'TLS' => 'Timor-Leste',
+          'TGO' => 'Togo',
+          'TKL' => 'Tokelau',
+          'TON' => 'Tonga',
+          'TTO' => 'Trinidad and Tobago',
+          'TUN' => 'Tunisia',
+          'TUR' => 'Turkey',
+          'TKM' => 'Turkmenistan',
+          'TCA' => 'Turks and Caicos Islands',
+          'TUV' => 'Tuvalu',
+          'UGA' => 'Uganda',
+          'UKR' => 'Ukraine',
+          'ARE' => 'United Arab Emirates',
+          'GBR' => 'United Kingdom of Great Britain and Northern Ireland',
+          'UMI' => 'United States Minor Outlying Islands',
+          'USA' => 'United States of America',
+          'URY' => 'Uruguay',
+          'UZB' => 'Uzbekistan',
+          'VUT' => 'Vanuatu',
+          'VEN' => 'Venezuela',
+          'VNM' => 'Viet Nam',
+          'VGB' => 'Virgin Islands',
+          'VIR' => 'Virgin Islands',
+          'WLF' => 'Wallis and Futuna',
+          'ESH' => 'Western Sahara',
+          'YEM' => 'Yemen',
+          'ZMB' => 'Zambia',
+          'ZWE' => 'Zimbabwe',
+          'ALA' => 'Åland Islands'
+        }
+      end
+    end
+  end
+end