portal_scraper 2.0.2 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b5b77cb5759492b2d73d62b20dc152bd3b28647c7be2212538c0f647fd5db5f4
|
4
|
+
data.tar.gz: 1e2d37f32f66e12b9853636684e02fa6f1d611773c85de8a97ed74533ff6f41a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a05dc807833a9668268335c68ee157c078b8ab52d316878b79b56c859fb14f6634f85332a24faabcc1895fb693643e2b1e1fc2819e4ad10048358f411d77fe72
|
7
|
+
data.tar.gz: e6657265707f3d4d3e8891332928ae5bb5b55fd7b60e43f6793ee3288470092663167a64398e263df7d3342f418394e443ce14be66ba84183cd3a0a13d59aaa8
|
@@ -9,7 +9,9 @@ module PortalScraper
|
|
9
9
|
attr_accessor :app_id, :secret
|
10
10
|
|
11
11
|
CODE_TO_COUNTRY_MAPPING = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'code_to_country.json')))
|
12
|
+
IS_POSTAL_CODE_DOM = JSON.parse(File.read(File.join(File.dirname(__FILE__), 'is_postal_code_dom.json')))
|
12
13
|
MINIMUM_AMOUNT = 10
|
14
|
+
PROPOSITION_INDEX = 5
|
13
15
|
|
14
16
|
def initialize(app_id = nil, secret = nil)
|
15
17
|
@app_id = app_id || config.app_id
|
@@ -24,8 +26,8 @@ module PortalScraper
|
|
24
26
|
user_page = app.post(url_for('/appRecherche.do'), { nom: '', prenom: '', chercher: 'Chercher' })
|
25
27
|
loop do
|
26
28
|
user_page.links_with(href: /appTiersDetail.do/).each do |link|
|
27
|
-
account
|
28
|
-
selected_link
|
29
|
+
account = link.click
|
30
|
+
selected_link = nil
|
29
31
|
opened_on, balance = nil, 0.to_d
|
30
32
|
|
31
33
|
account.links_with(css: 'table#contrat td a').each do |account_link|
|
@@ -36,19 +38,24 @@ module PortalScraper
|
|
36
38
|
next unless opened_on.nil? || opened_on < row_date
|
37
39
|
|
38
40
|
selected_link = account_link
|
39
|
-
opened_on
|
40
|
-
balance
|
41
|
+
opened_on = row_date
|
42
|
+
balance = parse_number(row.at('td[5]'))
|
41
43
|
end
|
42
44
|
|
43
45
|
next unless selected_link
|
44
46
|
|
45
47
|
transfers = scrap_transfers(selected_link)
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
account_data = {
|
49
|
+
client_ref: find_table_value(account, 'Identifiant tiers'),
|
50
|
+
balance: balance,
|
51
|
+
opened_on: opened_on,
|
52
|
+
transfers: transfers,
|
51
53
|
}
|
54
|
+
if block_given?
|
55
|
+
yield(account_data)
|
56
|
+
else
|
57
|
+
accounts_data[:accounts] << account_data
|
58
|
+
end
|
52
59
|
end
|
53
60
|
|
54
61
|
break unless user_page.search('.pg_next').present?
|
@@ -68,7 +75,7 @@ module PortalScraper
|
|
68
75
|
form['commune'] = user[:codeInsee]
|
69
76
|
form['libCommune'] = user[:ville]
|
70
77
|
else
|
71
|
-
fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville])
|
78
|
+
fill_commune(app, form, user[:codePostal], parse_city_name(user[:ville]))
|
72
79
|
end
|
73
80
|
fill_address(form, user[:numeroVoie])
|
74
81
|
fill_birth_place(form, user[:paysNaissance])
|
@@ -85,6 +92,8 @@ module PortalScraper
|
|
85
92
|
end
|
86
93
|
end
|
87
94
|
nil
|
95
|
+
rescue => e
|
96
|
+
{ error: e.message }
|
88
97
|
end
|
89
98
|
|
90
99
|
def create_proposition(client_ref)
|
@@ -123,7 +132,7 @@ module PortalScraper
|
|
123
132
|
end
|
124
133
|
|
125
134
|
def proposition
|
126
|
-
proposition_page = app.get(url_for('/propositionDetail.do'), { idx:
|
135
|
+
proposition_page = app.get(url_for('/propositionDetail.do'), { idx: PROPOSITION_INDEX })
|
127
136
|
proposition_form = proposition_page.form_with(name: /proposition/i)
|
128
137
|
try_form_fields(proposition_form, %w(montant amount), MINIMUM_AMOUNT)
|
129
138
|
proposition_form['regimeFiscal'] = 'I'
|
@@ -148,14 +157,15 @@ module PortalScraper
|
|
148
157
|
end
|
149
158
|
end
|
150
159
|
|
151
|
-
def fill_commune(app, form, postal_code, city
|
160
|
+
def fill_commune(app, form, postal_code, city)
|
152
161
|
app.post(url_for('/tiersCommunes.do'), { codPos: postal_code }).tap do |mappings|
|
153
|
-
cities
|
154
|
-
matched_city
|
162
|
+
cities = mappings.search('td.t').map { |c| { code: c.parent.search('a.PL_LST').text.strip, name: c.text.strip } }
|
163
|
+
matched_city = cities.find(-> { cities.first }) { |c| c[:name].include?(city) }
|
164
|
+
raise "City '#{city}' and postcode '#{postal_code}' not found" unless matched_city
|
155
165
|
form['commune'] = matched_city[:code]
|
156
166
|
form['libCommune'] = matched_city[:name]
|
157
167
|
end
|
158
|
-
form['territoireRes'] =
|
168
|
+
form['territoireRes'] = IS_POSTAL_CODE_DOM.dig(postal_code[0..2], 'category') || 0
|
159
169
|
end
|
160
170
|
|
161
171
|
def fill_address(form, address)
|
@@ -0,0 +1 @@
|
|
1
|
+
{"971":{"name":"GUADELOUPE","category":1},"972":{"name":"MARTINIQUE","category":1},"973":{"name":"GUYANE FRANCAISE","category":1},"974":{"name":"REUNION","category":1}}
|
data/portal-scraper.gemspec
CHANGED
@@ -23,11 +23,11 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.required_ruby_version = '>= 2.5.3'
|
24
24
|
|
25
25
|
spec.add_dependency 'mechanize', '~> 2.7'
|
26
|
-
spec.add_dependency 'activesupport', '
|
26
|
+
spec.add_dependency 'activesupport', '>= 5.2', '< 7.0'
|
27
27
|
|
28
28
|
spec.add_development_dependency 'rspec', '~> 3.2'
|
29
29
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
30
|
-
spec.add_development_dependency 'rake', '~>
|
30
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
31
31
|
spec.add_development_dependency 'vcr'
|
32
32
|
spec.add_development_dependency 'webmock'
|
33
33
|
spec.add_development_dependency 'rubocop', '~> 0.67.2'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: portal_scraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Armand Mégrot
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-10-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mechanize
|
@@ -29,16 +29,22 @@ dependencies:
|
|
29
29
|
name: activesupport
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
|
-
- - "
|
32
|
+
- - ">="
|
33
33
|
- !ruby/object:Gem::Version
|
34
34
|
version: '5.2'
|
35
|
+
- - "<"
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '7.0'
|
35
38
|
type: :runtime
|
36
39
|
prerelease: false
|
37
40
|
version_requirements: !ruby/object:Gem::Requirement
|
38
41
|
requirements:
|
39
|
-
- - "
|
42
|
+
- - ">="
|
40
43
|
- !ruby/object:Gem::Version
|
41
44
|
version: '5.2'
|
45
|
+
- - "<"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '7.0'
|
42
48
|
- !ruby/object:Gem::Dependency
|
43
49
|
name: rspec
|
44
50
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,14 +79,14 @@ dependencies:
|
|
73
79
|
requirements:
|
74
80
|
- - "~>"
|
75
81
|
- !ruby/object:Gem::Version
|
76
|
-
version: '
|
82
|
+
version: '13.0'
|
77
83
|
type: :development
|
78
84
|
prerelease: false
|
79
85
|
version_requirements: !ruby/object:Gem::Requirement
|
80
86
|
requirements:
|
81
87
|
- - "~>"
|
82
88
|
- !ruby/object:Gem::Version
|
83
|
-
version: '
|
89
|
+
version: '13.0'
|
84
90
|
- !ruby/object:Gem::Dependency
|
85
91
|
name: vcr
|
86
92
|
requirement: !ruby/object:Gem::Requirement
|
@@ -148,6 +154,7 @@ files:
|
|
148
154
|
- lib/portal_scraper.rb
|
149
155
|
- lib/portal_scraper/accounts/client.rb
|
150
156
|
- lib/portal_scraper/accounts/code_to_country.json
|
157
|
+
- lib/portal_scraper/accounts/is_postal_code_dom.json
|
151
158
|
- lib/portal_scraper/bad_login_url.rb
|
152
159
|
- lib/portal_scraper/bad_root_url.rb
|
153
160
|
- lib/portal_scraper/parsing_helper.rb
|