base_scraper_service 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69eb49ba30aa10adba230399397bc900df7f46b10da8c5ef7e02d00743fd7407
-  data.tar.gz: a7534672884d78323aa41838e1048c1cfff7a59d19a7f633034eaf9e8dc680d0
+  metadata.gz: 75c1ba4ef16ea7a756bae017e95051e3c572c55d169da8d4cb7331bdf12c974b
+  data.tar.gz: fe704e0af988b43de06b235bdee1761a9726cbb153a0b8cbdc22dab770ae1960
 SHA512:
-  metadata.gz: b8957504e31a3a5519538bd7b5fecdb7837f502b5cdccea51c59d3c585bb26f5dc7642449808e3913d361ee778af74e06708dc54b5ed771cb8839434f1e7834c
-  data.tar.gz: 6227de3f572c770425145ef074dbbdbec0cebf8cd0c7f95ba4cc1d00a0ef24e60534e3537d5ae305d1a6af41688304b55fea00b4bd78b6a3b0e951ee5f28dea4
+  metadata.gz: ca95405e99c42f18556b91c8b48db8c173c10f2305fb55465644c85c408b2492216978329c47f6cc4bd5273c36c2e8811b5362dc66bc3fbc46ba782079d0a14a
+  data.tar.gz: 51d1fed725886178366a038cae7a5386dd3bc389e490f4a537f6c91aaf39a441fe39d99932a114d22aaa3dbd11958eb2df0fe0143d5a6d18f177588974f7ba36

data/lib/base_scraper_service/agent_object.rb

@@ -1,6 +1,5 @@
 require_relative 'user_agent'
 require "mechanize"
-require "nokogiri"
 
 module BaseScraper
   class Service

data/lib/base_scraper_service/base.rb

@@ -1,5 +1,4 @@
 require 'sinatra'
-require 'dotenv'
 require "./#{ENV['SERVICE_NAME']}_offers_service"
 
 configure { set :server, :puma }

data/lib/base_scraper_service/base_scraper.rb

@@ -1,16 +1,6 @@
-require 'mechanize'
-require 'nokogiri'
-require 'thread'
-require 'fileutils'
-require 'logger'
-require "dotenv"
 require 'csv'
-require 'date'
 require 'active_support/time'
 require 'library_stdnums'
-require 'base_scraper_service'
-require_relative 'user_agent'
-require_relative 'agent_object'
 
 module BaseScraper
   class Service

data/lib/base_scraper_service/location_service.rb

@@ -9,29 +9,34 @@ module BaseScraper
         # 1- return state abbreviation if matched with state abbreviations
         # 2- return state abbreviation if matched with state name
         # 3- otherwise return empty string
-        state = us_states_hash.keys.select { |state_abbreviation| @address.upcase.include?(state_abbreviation) }.first
-        return state if state.present? && @address.length != 3
-        state = us_states_hash.select { |_, state| @address.include?(state) }.first&.first
 
-        return state.to_s
+        @state = us_states_hash.keys.select { |state_abbreviation| @address.upcase.include?(state_abbreviation) }.first
+        return @state if @state.present? && @address.length != 3
+        @state = us_states_hash.select { |_, state| @address.include?(state) }.first&.first
+
+        return @state.to_s
+
       end
 
+
+
       def extract_country
         # 1- return country abbreviation if matched with country abbreviations
         # 2- return country abbreviation if matched with country name
         # 3- return USA if address contains a USA state
         # 4- otherwise return empty string
+
         country = countries_hash.keys.select { |country_abbreviation| @address.include?(country_abbreviation) }.first
-        puts country, @address
         return country if country.present?
-        country = countries_hash.select { |_, country_name| @address.include?(country_name) }.first&.first
+        country = countries_hash.select { |_, country_name| @address.downcase.include?(country_name.downcase) }.first&.first
         return country if country.present?
 
-        return 'USA' if extract_state.present?
+        return 'USA' if @state.present?
 
         return country.to_s
       end
 
+
       private
 
       def us_states_hash
@@ -327,7 +332,7 @@ module BaseScraper
           'ARE' => 'United Arab Emirates',
           'GBR' => 'United Kingdom of Great Britain and Northern Ireland',
           'UMI' => 'United States Minor Outlying Islands',
-          'USA' => 'United States of America',
+          'USA' => 'United States',
           'URY' => 'Uruguay',
           'UZB' => 'Uzbekistan',
           'VUT' => 'Vanuatu',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: base_scraper_service
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Qbatch
@@ -38,76 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: thread
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: fileutils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: logger
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: date
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: dotenv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -150,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: library_stdnums
   requirement: !ruby/object:Gem::Requirement
@@ -178,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: shotgun
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
@@ -206,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Base scraper service to handle multiple services
 email: 
 executables: []