portal_scraper 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31516d14a7a801b838c845184d8c561170532c39cedb017efe9c543fdf5ecc9c
-  data.tar.gz: 7858244b3bfca539c9303f8bad9e0cfaaa0bfe1116ca34f67782ee97f112ffad
+  metadata.gz: 3f511d4ae5d6568df6bc6960907a0964ecfb6c8f0c8d5850b5ad757a0ed35039
+  data.tar.gz: 5edb4fdbd33676d7b03623a86815b0b865fd214d85e4cd6e34e7fe42fa67204b
 SHA512:
-  metadata.gz: 8ca16dace317cacf3f1351d7009c29ef1231341372f764c34182f866727ca2e155fca3f90487d9d3a36ef4e296a12db5be12deef452288f5d4d18c2558059c86
-  data.tar.gz: fe54374b24f9252cc4186208b5947d5de58ddf77c9541a868774eedae939a3e02218466f23b08398722948e6b85c0c917661fd3a68c7e7e1b2e2dc698902cc80
+  metadata.gz: 5df684726b58e1d8da72af7abd8b0395d36bc403f8d045ef0abbf628a8eee0aeebaa13f13ecbc9e80b9ff1dd088695eec92470dac707d5e5977dab1961196234
+  data.tar.gz: d93c42d08e8f39ddacf8828060e91324f0ad1b42f1f131b381d43ba693bbba8d05c3d82e1c669736fc153d087b9f648ba1af8fab7936be25c70ee10d811c2708

data/lib/portal_scraper/accounts/client.rb

@@ -21,35 +21,35 @@ module PortalScraper
         login
         accounts_data = { accounts: [] }
 
-        results = app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
+        user_page = app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
         loop do
-          results.links_with(href: /appTiersDetail.do/).each do |link|
-            account   = link.click
-            transfers = []
-            account.links_with(css: 'table#contrat td a').each do |transfer|
-              result = transfer.click
-              result.search('table#operation tbody tr').each do |row|
-                transfers << {
-                  operation_date: parse_date(row.search('td[1]')),
-                  nature:         row.search('td[2]').text.strip.gsub(/[\n,\t,'']/, ''),
-                  value_date:     parse_date(row.search('td[3]')),
-                  debit:          parse_number(row.search('td[4]')),
-                  credit:         parse_number(row.search('td[5]')),
-                }
+          user_page.links_with(href: /appTiersDetail.do/).each do |link|
+            account = link.click
+            selected_row = nil
+            selected_row_date = nil
+
+            account.links_with(css: 'table#contrat td a').each do |row|
+              row_date = parse_date(row.node.ancestors('tr').at('td[2]'))&.to_date
+              if selected_row_date.nil? || selected_row_date < row_date
+                selected_row_date = row_date
+                selected_row = row
               end
             end
+
+            transfers = scrap_transfers(selected_row)
+            balance = selected_row&.node&.ancestors('tr')&.at('td[4]')
             accounts_data[:accounts] << {
               client_ref: find_table_value(account, 'Identifiant tiers'),
-              balance:    parse_number(account.search('table#contrat th.n')),
-              opened_on:  parse_dates(account.search('table#contrat td'))&.max,
+              balance:    balance.nil? ? "0".to_d : parse_number(balance),
+              opened_on:  selected_row_date.nil? ? parse_dates(account.search('table#contrat td'))&.max : selected_row_date.to_s,
               transfers:  transfers,
             }
           end
 
-          break unless results.search('.pg_next').present?
+          break unless user_page.search('.pg_next').present?
           # Need to fetch the first page as you cannot navigate from client back to research page
           app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
-          results = app.get(results.search('a.pg_next').first['href'])
+          user_page = app.get(user_page.search('a.pg_next').first['href'])
         end
         accounts_data
       end
@@ -174,6 +174,23 @@ module PortalScraper
       def config
         PortalScraper.config
       end
+
+      def scrap_transfers(selected_row)
+        transfers = []
+        transfers_page = selected_row&.click
+        if transfers_page
+          transfers_page.search('table#operation tbody tr').each do |row|
+            transfers << {
+                operation_date: parse_date(row.search('td[1]')),
+                nature:         row.search('td[2]').text.strip.gsub(/[\n,\t,'']/, ''),
+                value_date:     parse_date(row.search('td[3]')),
+                debit:          parse_number(row.search('td[4]')),
+                credit:         parse_number(row.search('td[5]')),
+            }
+          end
+        end
+        transfers
+      end
     end
   end
 end

data/lib/portal_scraper/version.rb

@@ -2,7 +2,7 @@ module PortalScraper
   class Version
     MAJOR = 2
     MINOR = 0
-    PATCH = 0
+    PATCH = 1
 
     def self.to_s
       [MAJOR, MINOR, PATCH].compact.join('.')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: portal_scraper
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Armand Mégrot
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-05 00:00:00.000000000 Z
+date: 2020-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize