portal_scraper 2.0.2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4da909bd4c96e2f086b1c3d829c27a1ed6be1fc2dd2a071d1b0c95aaf062f60a
-  data.tar.gz: dd4c79ee59b02dc544b54507cb74a67203f17f661c39add242024a82c52dd837
+  metadata.gz: b5b77cb5759492b2d73d62b20dc152bd3b28647c7be2212538c0f647fd5db5f4
+  data.tar.gz: 1e2d37f32f66e12b9853636684e02fa6f1d611773c85de8a97ed74533ff6f41a
 SHA512:
-  metadata.gz: ad8b34815e11e7eea8cfd7278b0fe5ad9342567855fdef8faba6a274ca0731f3976aed8804372889f742f6d8f10b72730e55f4e6ce3b18de4b24d8c6f48fa1c9
-  data.tar.gz: ca74f2b24e0acf074795028ed4b84a0190197743d3c30b96f6b8b13873f6cf0938b775f403c4fc878f99600cede3f39db0f3a288a9f08e5d7a473a0df6969d93
+  metadata.gz: a05dc807833a9668268335c68ee157c078b8ab52d316878b79b56c859fb14f6634f85332a24faabcc1895fb693643e2b1e1fc2819e4ad10048358f411d77fe72
+  data.tar.gz: e6657265707f3d4d3e8891332928ae5bb5b55fd7b60e43f6793ee3288470092663167a64398e263df7d3342f418394e443ce14be66ba84183cd3a0a13d59aaa8

data/lib/portal_scraper/accounts/client.rb

@@ -9,7 +9,9 @@ module PortalScraper
       attr_accessor :app_id, :secret
 
       CODE_TO_COUNTRY_MAPPING = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'code_to_country.json')))
+      IS_POSTAL_CODE_DOM      = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'is_postal_code_dom.json')))
       MINIMUM_AMOUNT          = 10
+      PROPOSITION_INDEX       = 5
 
       def initialize(app_id = nil, secret = nil)
         @app_id = app_id || config.app_id
@@ -24,8 +26,8 @@ module PortalScraper
         user_page = app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
         loop do
           user_page.links_with(href: /appTiersDetail.do/).each do |link|
-            account = link.click
-            selected_link = nil
+            account            = link.click
+            selected_link      = nil
             opened_on, balance = nil, 0.to_d
 
             account.links_with(css: 'table#contrat td a').each do |account_link|
@@ -36,19 +38,24 @@ module PortalScraper
               next unless opened_on.nil? || opened_on < row_date
 
               selected_link = account_link
-              opened_on = row_date
-              balance = parse_number(row.at('td[4]'))
+              opened_on     = row_date
+              balance       = parse_number(row.at('td[5]'))
             end
 
             next unless selected_link
 
             transfers = scrap_transfers(selected_link)
-            accounts_data[:accounts] << {
-              client_ref: find_table_value(account, 'Identifiant tiers'),
-              balance:    balance,
-              opened_on:  opened_on,
-              transfers:  transfers,
+            account_data = {
+                client_ref: find_table_value(account, 'Identifiant tiers'),
+                balance:    balance,
+                opened_on:  opened_on,
+                transfers:  transfers,
             }
+            if block_given?
+              yield(account_data)
+            else
+              accounts_data[:accounts] << account_data
+            end
           end
 
           break unless user_page.search('.pg_next').present?
@@ -68,7 +75,7 @@ module PortalScraper
               form['commune']    = user[:codeInsee]
               form['libCommune'] = user[:ville]
             else
-              fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville]), user[:pays] || user[:libNation])
+              fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville]))
             end
             fill_address(form, user[:numeroVoie])
             fill_birth_place(form, user[:paysNaissance])
@@ -85,6 +92,8 @@ module PortalScraper
           end
         end
         nil
+      rescue => e
+        { error: e.message }
       end
 
       def create_proposition(client_ref)
@@ -123,7 +132,7 @@ module PortalScraper
       end
 
       def proposition
-        proposition_page = app.get(url_for('/propositionDetail.do'), { idx: 3 })
+        proposition_page = app.get(url_for('/propositionDetail.do'), { idx: PROPOSITION_INDEX })
         proposition_form = proposition_page.form_with(name: /proposition/i)
         try_form_fields(proposition_form, %w(montant amount), MINIMUM_AMOUNT)
         proposition_form['regimeFiscal'] = 'I'
@@ -148,14 +157,15 @@ module PortalScraper
         end
       end
 
-      def fill_commune(app, form, postal_code, city, country_code)
+      def fill_commune(app, form, postal_code, city)
         app.post(url_for('/tiersCommunes.do'), { codPos: postal_code }).tap do |mappings|
-          cities             = mappings.search('td.t').map { |c| { code: c.parent.search('a.PL_LST').text.strip, name: c.text.strip } }
-          matched_city       = cities.find(-> { cities.first }) { |c| c[:name].include?(city) }
+          cities       = mappings.search('td.t').map { |c| { code: c.parent.search('a.PL_LST').text.strip, name: c.text.strip } }
+          matched_city = cities.find(-> { cities.first }) { |c| c[:name].include?(city) }
+          raise "City '#{city}' and postcode '#{postal_code}' not found" unless matched_city
           form['commune']    = matched_city[:code]
           form['libCommune'] = matched_city[:name]
         end
-        form['territoireRes'] = CODE_TO_COUNTRY_MAPPING.dig(country_code, 'category')
+        form['territoireRes'] = IS_POSTAL_CODE_DOM.dig(postal_code[0..2], 'category') || 0
       end
 
       def fill_address(form, address)

data/lib/portal_scraper/accounts/is_postal_code_dom.json

@@ -0,0 +1 @@
+{"971":{"name":"GUADELOUPE","category":1},"972":{"name":"MARTINIQUE","category":1},"973":{"name":"GUYANE FRANCAISE","category":1},"974":{"name":"REUNION","category":1}}

data/lib/portal_scraper/version.rb

@@ -1,8 +1,8 @@
 module PortalScraper
   class Version
     MAJOR = 2
-    MINOR = 0
-    PATCH = 2
+    MINOR = 3
+    PATCH = 0
 
     def self.to_s
       [MAJOR, MINOR, PATCH].compact.join('.')

data/portal-scraper.gemspec

@@ -23,11 +23,11 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.5.3'
 
   spec.add_dependency 'mechanize', '~> 2.7'
-  spec.add_dependency 'activesupport', '~> 5.2'
+  spec.add_dependency 'activesupport', '>= 5.2', '< 7.0'
 
   spec.add_development_dependency 'rspec', '~> 3.2'
   spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'webmock'
   spec.add_development_dependency 'rubocop', '~> 0.67.2'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: portal_scraper
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.3.0
 platform: ruby
 authors:
 - Armand Mégrot
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -29,16 +29,22 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -73,14 +79,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -148,6 +154,7 @@ files:
 - lib/portal_scraper.rb
 - lib/portal_scraper/accounts/client.rb
 - lib/portal_scraper/accounts/code_to_country.json
+- lib/portal_scraper/accounts/is_postal_code_dom.json
 - lib/portal_scraper/bad_login_url.rb
 - lib/portal_scraper/bad_root_url.rb
 - lib/portal_scraper/parsing_helper.rb