portal_scraper 2.0.1 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f511d4ae5d6568df6bc6960907a0964ecfb6c8f0c8d5850b5ad757a0ed35039
-  data.tar.gz: 5edb4fdbd33676d7b03623a86815b0b865fd214d85e4cd6e34e7fe42fa67204b
+  metadata.gz: 0be4e772a2a12824b3b8758f9914f89fa1403100749ba81b4f6a73e036c21064
+  data.tar.gz: b1cd8df8f5e4427b53c2e86a498f9bb141175d8cd844e3b9df64125ce4e8cdc2
 SHA512:
-  metadata.gz: 5df684726b58e1d8da72af7abd8b0395d36bc403f8d045ef0abbf628a8eee0aeebaa13f13ecbc9e80b9ff1dd088695eec92470dac707d5e5977dab1961196234
-  data.tar.gz: d93c42d08e8f39ddacf8828060e91324f0ad1b42f1f131b381d43ba693bbba8d05c3d82e1c669736fc153d087b9f648ba1af8fab7936be25c70ee10d811c2708
+  metadata.gz: 9c1f8dca1fb37272f1f67da43b050d594b4ebb79d2ad377ca18bbae72909dd78095a15e634006197bc150e4720c9b78e2ee4ebb00799145c2de706daf0be0775
+  data.tar.gz: ddce7d5af7db47a15fdbc4b0abe118e3636f218497cb61ca7e3ba3197e3b539a2916661b4de84c5d6880b1a646e83b97d4635a43973f254df79bd194085a89d9

data/lib/portal_scraper/accounts/client.rb

@@ -9,6 +9,7 @@ module PortalScraper
       attr_accessor :app_id, :secret
 
       CODE_TO_COUNTRY_MAPPING = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'code_to_country.json')))
+      IS_POSTAL_CODE_DOM      = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'is_postal_code_dom.json')))
       MINIMUM_AMOUNT          = 10
 
       def initialize(app_id = nil, secret = nil)
@@ -24,26 +25,36 @@ module PortalScraper
         user_page = app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
         loop do
           user_page.links_with(href: /appTiersDetail.do/).each do |link|
-            account = link.click
-            selected_row = nil
-            selected_row_date = nil
-
-            account.links_with(css: 'table#contrat td a').each do |row|
-              row_date = parse_date(row.node.ancestors('tr').at('td[2]'))&.to_date
-              if selected_row_date.nil? || selected_row_date < row_date
-                selected_row_date = row_date
-                selected_row = row
-              end
+            account            = link.click
+            selected_link      = nil
+            opened_on, balance = nil, 0.to_d
+
+            account.links_with(css: 'table#contrat td a').each do |account_link|
+              row = account_link.node.ancestors('tr')
+              next unless row.at('td[1]').text.strip == 'Compte épargne rémunéré'
+
+              row_date = parse_date(row.at('td[2]'))
+              next unless opened_on.nil? || opened_on < row_date
+
+              selected_link = account_link
+              opened_on     = row_date
+              balance       = parse_number(row.at('td[5]'))
             end
 
-            transfers = scrap_transfers(selected_row)
-            balance = selected_row&.node&.ancestors('tr')&.at('td[4]')
-            accounts_data[:accounts] << {
-              client_ref: find_table_value(account, 'Identifiant tiers'),
-              balance:    balance.nil? ? "0".to_d : parse_number(balance),
-              opened_on:  selected_row_date.nil? ? parse_dates(account.search('table#contrat td'))&.max : selected_row_date.to_s,
-              transfers:  transfers,
+            next unless selected_link
+
+            transfers = scrap_transfers(selected_link)
+            account_data = {
+                client_ref: find_table_value(account, 'Identifiant tiers'),
+                balance:    balance,
+                opened_on:  opened_on,
+                transfers:  transfers,
             }
+            if block_given?
+              yield(account_data)
+            else
+              accounts_data[:accounts] << account_data
+            end
           end
 
           break unless user_page.search('.pg_next').present?
@@ -63,7 +74,7 @@ module PortalScraper
               form['commune']    = user[:codeInsee]
               form['libCommune'] = user[:ville]
             else
-              fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville]), user[:pays] || user[:libNation])
+              fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville]))
             end
             fill_address(form, user[:numeroVoie])
             fill_birth_place(form, user[:paysNaissance])
@@ -80,6 +91,8 @@ module PortalScraper
           end
         end
         nil
+      rescue => e
+        { error: e.message }
       end
 
       def create_proposition(client_ref)
@@ -143,14 +156,15 @@ module PortalScraper
         end
       end
 
-      def fill_commune(app, form, postal_code, city, country_code)
+      def fill_commune(app, form, postal_code, city)
         app.post(url_for('/tiersCommunes.do'), { codPos: postal_code }).tap do |mappings|
-          cities             = mappings.search('td.t').map { |c| { code: c.parent.search('a.PL_LST').text.strip, name: c.text.strip } }
-          matched_city       = cities.find(-> { cities.first }) { |c| c[:name].include?(city) }
+          cities       = mappings.search('td.t').map { |c| { code: c.parent.search('a.PL_LST').text.strip, name: c.text.strip } }
+          matched_city = cities.find(-> { cities.first }) { |c| c[:name].include?(city) }
+          raise "City '#{city}' and postcode '#{postal_code}' not found" unless matched_city
           form['commune']    = matched_city[:code]
           form['libCommune'] = matched_city[:name]
         end
-        form['territoireRes'] = CODE_TO_COUNTRY_MAPPING.dig(country_code, 'category')
+        form['territoireRes'] = IS_POSTAL_CODE_DOM.dig(postal_code[0..2], 'category') || 0
       end
 
       def fill_address(form, address)
@@ -175,21 +189,17 @@ module PortalScraper
         PortalScraper.config
       end
 
-      def scrap_transfers(selected_row)
-        transfers = []
-        transfers_page = selected_row&.click
-        if transfers_page
-          transfers_page.search('table#operation tbody tr').each do |row|
-            transfers << {
-                operation_date: parse_date(row.search('td[1]')),
-                nature:         row.search('td[2]').text.strip.gsub(/[\n,\t,'']/, ''),
-                value_date:     parse_date(row.search('td[3]')),
-                debit:          parse_number(row.search('td[4]')),
-                credit:         parse_number(row.search('td[5]')),
-            }
-          end
+      def scrap_transfers(account_link)
+        transfers_page = account_link.click
+        transfers_page.search('table#operation tbody tr').map do |transfer_row|
+          {
+              operation_date: parse_date(transfer_row.search('td[1]')),
+              nature:         transfer_row.search('td[2]').text.strip.gsub(/[\n,\t,'']/, ''),
+              value_date:     parse_date(transfer_row.search('td[3]')),
+              debit:          parse_number(transfer_row.search('td[4]')),
+              credit:         parse_number(transfer_row.search('td[5]')),
+          }
         end
-        transfers
       end
     end
   end

data/lib/portal_scraper/accounts/is_postal_code_dom.json

@@ -0,0 +1 @@
+{"971":{"name":"GUADELOUPE","category":1},"972":{"name":"MARTINIQUE","category":1},"973":{"name":"GUYANE FRANCAISE","category":1},"974":{"name":"REUNION","category":1}}

data/lib/portal_scraper/parsing_helper.rb

@@ -13,7 +13,7 @@ module PortalScraper
     end
 
     def parse_dates(node)
-      node.text.strip.gsub(%r(\d{2}/\d{2}/\d{4}))
+      node.text.strip.gsub(%r(\d{2}/\d{2}/\d{4})).map { |d| Date.strptime(d, '%d/%m/%Y') }
     end
   end
 end

data/lib/portal_scraper/version.rb

@@ -1,7 +1,7 @@
 module PortalScraper
   class Version
     MAJOR = 2
-    MINOR = 0
+    MINOR = 2
     PATCH = 1
 
     def self.to_s

data/portal-scraper.gemspec

@@ -23,11 +23,11 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.5.3'
 
   spec.add_dependency 'mechanize', '~> 2.7'
-  spec.add_dependency 'activesupport', '~> 5.2'
+  spec.add_dependency 'activesupport', '>= 5.2', '< 7.0'
 
   spec.add_development_dependency 'rspec', '~> 3.2'
   spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'webmock'
   spec.add_development_dependency 'rubocop', '~> 0.67.2'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: portal_scraper
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.2.1
 platform: ruby
 authors:
 - Armand Mégrot
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-06 00:00:00.000000000 Z
+date: 2020-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -29,16 +29,22 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -73,14 +79,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -148,6 +154,7 @@ files:
 - lib/portal_scraper.rb
 - lib/portal_scraper/accounts/client.rb
 - lib/portal_scraper/accounts/code_to_country.json
+- lib/portal_scraper/accounts/is_postal_code_dom.json
 - lib/portal_scraper/bad_login_url.rb
 - lib/portal_scraper/bad_root_url.rb
 - lib/portal_scraper/parsing_helper.rb