ehbrs-tools 0.28.2 → 0.28.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +51 -17
- data/lib/ehbrs/tools/runner/fs/selected.rb +1 -1
- data/lib/ehbrs/tools/runner/vg/wii.rb +2 -2
- data/lib/ehbrs/tools/runner/videos/series/rename.rb +1 -1
- data/lib/ehbrs/tools/version.rb +1 -1
- data/lib/ehbrs/videos/unsupported/search.rb +2 -2
- data/vendor/aranha-parsers/aranha-parsers.gemspec +1 -0
- data/vendor/aranha-parsers/lib/aranha/parsers/base.rb +3 -1
- data/vendor/aranha-parsers/lib/aranha/parsers/html/node/default/numeric_support.rb +70 -0
- data/vendor/aranha-parsers/lib/aranha/parsers/html/node/default/string_support.rb +77 -0
- data/vendor/aranha-parsers/lib/aranha/parsers/html/node/default.rb +2 -94
- data/vendor/aranha-parsers/lib/aranha/parsers/json/base.rb +20 -0
- data/vendor/aranha-parsers/lib/aranha/parsers/json.rb +11 -0
- data/vendor/aranha-parsers/lib/aranha/parsers/source_address/http_get.rb +12 -15
- data/vendor/aranha-parsers/lib/aranha/parsers/version.rb +1 -1
- data/vendor/eac_fs/lib/eac_fs/patches/object/fs_cache.rb +4 -2
- data/vendor/eac_fs/lib/eac_fs/traversable.rb +45 -0
- data/vendor/eac_fs/lib/eac_fs/traverser.rb +76 -0
- data/vendor/eac_fs/lib/eac_fs/version.rb +1 -1
- data/vendor/eac_ruby_base0/lib/eac_ruby_base0/application.rb +1 -1
- data/vendor/eac_ruby_base0/lib/eac_ruby_base0/runner_with/confirmation.rb +0 -1
- data/vendor/eac_ruby_base0/lib/eac_ruby_base0/runner_with/filesystem_traverser.rb +2 -2
- data/vendor/eac_ruby_base0/lib/eac_ruby_base0/version.rb +1 -1
- data/vendor/eac_ruby_gems_utils/eac_ruby_gems_utils.gemspec +1 -1
- data/vendor/eac_ruby_gems_utils/lib/eac_ruby_gems_utils/tests/base.rb +0 -1
- data/vendor/eac_ruby_gems_utils/lib/eac_ruby_gems_utils/version.rb +1 -1
- data/vendor/eac_ruby_gems_utils/spec/support/mygem/Gemfile.lock +1 -1
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/abstract_methods.rb +7 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/core_ext.rb +1 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/gems_registry/gem.rb +4 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/gems_registry.rb +1 -1
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/patches/class/abstract.rb +9 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/patches/class/self_included_modules.rb +7 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/patches/module/i18n_translate.rb +32 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/patches/object/call_if_proc.rb +11 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/patches/object/i18n_translate.rb +13 -0
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/speaker/receiver.rb +10 -10
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/version.rb +1 -1
- data/vendor/eac_ruby_utils/spec/lib/eac_ruby_utils/patches/class/abstract_spec.rb +25 -0
- data/vendor/eac_ruby_utils/spec/lib/eac_ruby_utils/patches/class/self_included_modules_spec.rb +30 -0
- data/vendor/eac_ruby_utils/spec/lib/eac_ruby_utils/patches/object/call_if_proc_spec.rb +11 -0
- data/vendor/ehbrs_ruby_utils/lib/ehbrs_ruby_utils/fs/selected.rb +11 -1
- data/vendor/ehbrs_ruby_utils/lib/ehbrs_ruby_utils/music/lyrics_book.rb +1 -1
- data/vendor/ehbrs_ruby_utils/lib/ehbrs_ruby_utils/version.rb +1 -1
- metadata +16 -5
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/fs/traversable.rb +0 -47
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/fs/traverser.rb +0 -72
- data/vendor/eac_ruby_utils/lib/eac_ruby_utils/on_clean_ruby_environment.rb +0 -12
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 644c4ba64fc1a688a62d35b89e6aadb311239b33611e782556931a6b3cf185b5
|
|
4
|
+
data.tar.gz: a8f8e6f90cdf1746b0996b0c1330f86559ba7bcf62c57df7015103a48a946043
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b6fb53198617263ae78fab68ebeae06ab0cb311c2353675defe9dc857d07fff5c93eb237b970910d2f7aa520521b342a74477908ad88dcaf4eb5f6122b00952d
|
|
7
|
+
data.tar.gz: c41b33ff7bcac502642466954e89ede742a2aca4a7cde650602476ac1bf3602f7756972c12593e346ff274744c3512a16a5604bf60de5ff22616bd242cc80984
|
data/Gemfile.lock
CHANGED
|
@@ -7,7 +7,7 @@ GIT
|
|
|
7
7
|
PATH
|
|
8
8
|
remote: .
|
|
9
9
|
specs:
|
|
10
|
-
ehbrs-tools (0.28.
|
|
10
|
+
ehbrs-tools (0.28.3)
|
|
11
11
|
avm-tools (~> 0.82, >= 0.82.1)
|
|
12
12
|
eac_cli (~> 0.12, >= 0.12.5)
|
|
13
13
|
eac_ruby_utils (~> 0.71)
|
|
@@ -20,11 +20,12 @@ PATH
|
|
|
20
20
|
PATH
|
|
21
21
|
remote: vendor/aranha-parsers
|
|
22
22
|
specs:
|
|
23
|
-
aranha-parsers (0.
|
|
23
|
+
aranha-parsers (0.10.0)
|
|
24
24
|
activesupport (>= 4.0.0)
|
|
25
25
|
addressable (~> 2.7)
|
|
26
26
|
curb (~> 0.9.10)
|
|
27
27
|
eac_ruby_utils (~> 0.74)
|
|
28
|
+
faraday_middleware
|
|
28
29
|
httpclient (~> 2.8, >= 2.8.3)
|
|
29
30
|
nokogiri (~> 1.12, >= 1.12.4)
|
|
30
31
|
ofx-parser (~> 1.1.0)
|
|
@@ -61,7 +62,7 @@ PATH
|
|
|
61
62
|
PATH
|
|
62
63
|
remote: vendor/eac_fs
|
|
63
64
|
specs:
|
|
64
|
-
eac_fs (0.
|
|
65
|
+
eac_fs (0.7.0)
|
|
65
66
|
content-type
|
|
66
67
|
eac_ruby_utils (~> 0.70)
|
|
67
68
|
ruby-filemagic
|
|
@@ -69,7 +70,7 @@ PATH
|
|
|
69
70
|
PATH
|
|
70
71
|
remote: vendor/eac_ruby_base0
|
|
71
72
|
specs:
|
|
72
|
-
eac_ruby_base0 (0.
|
|
73
|
+
eac_ruby_base0 (0.13.1)
|
|
73
74
|
eac_cli (~> 0.22, >= 0.22.2)
|
|
74
75
|
eac_fs (~> 0.3)
|
|
75
76
|
eac_ruby_gems_utils (~> 0.9.6, ~> 0.9)
|
|
@@ -78,14 +79,14 @@ PATH
|
|
|
78
79
|
PATH
|
|
79
80
|
remote: vendor/eac_ruby_gems_utils
|
|
80
81
|
specs:
|
|
81
|
-
eac_ruby_gems_utils (0.9.
|
|
82
|
-
bundler
|
|
82
|
+
eac_ruby_gems_utils (0.9.8)
|
|
83
|
+
bundler
|
|
83
84
|
eac_ruby_utils (~> 0.70)
|
|
84
85
|
|
|
85
86
|
PATH
|
|
86
87
|
remote: vendor/eac_ruby_utils
|
|
87
88
|
specs:
|
|
88
|
-
eac_ruby_utils (0.
|
|
89
|
+
eac_ruby_utils (0.79.0)
|
|
89
90
|
activesupport (>= 4.0)
|
|
90
91
|
addressable (~> 2.6)
|
|
91
92
|
filesize
|
|
@@ -101,7 +102,7 @@ PATH
|
|
|
101
102
|
PATH
|
|
102
103
|
remote: vendor/ehbrs_ruby_utils
|
|
103
104
|
specs:
|
|
104
|
-
ehbrs_ruby_utils (0.12.
|
|
105
|
+
ehbrs_ruby_utils (0.12.2)
|
|
105
106
|
aranha-parsers (~> 0.8, >= 0.8.5)
|
|
106
107
|
avm (~> 0.3, >= 0.3.2)
|
|
107
108
|
eac_fs (~> 0.5)
|
|
@@ -130,10 +131,11 @@ GEM
|
|
|
130
131
|
public_suffix (>= 2.0.2, < 5.0)
|
|
131
132
|
asciidoctor (2.0.16)
|
|
132
133
|
ast (2.4.2)
|
|
133
|
-
avm (0.
|
|
134
|
+
avm (0.8.0)
|
|
135
|
+
eac_cli (~> 0.23, >= 0.23.1)
|
|
134
136
|
eac_docker (~> 0.3)
|
|
135
137
|
eac_git (~> 0.6)
|
|
136
|
-
eac_ruby_utils (~> 0.
|
|
138
|
+
eac_ruby_utils (~> 0.77)
|
|
137
139
|
eac_templates (~> 0.1, >= 0.1.1)
|
|
138
140
|
filesize (~> 0.2)
|
|
139
141
|
htmlbeautifier (~> 1.3, >= 1.3.1)
|
|
@@ -144,13 +146,43 @@ GEM
|
|
|
144
146
|
eac_ruby_utils (~> 0.63)
|
|
145
147
|
os
|
|
146
148
|
rouge (~> 3.26)
|
|
149
|
+
avm-eac_rails_base1 (0.1.2)
|
|
150
|
+
avm-eac_webapp_base0 (~> 0.1)
|
|
151
|
+
eac_ruby_utils (~> 0.68)
|
|
152
|
+
avm-eac_redmine_base0 (0.3.0)
|
|
153
|
+
avm-eac_rails_base1 (~> 0.1)
|
|
154
|
+
avm-eac_ubuntu_base0 (~> 0.2)
|
|
155
|
+
curb (~> 0.9.10)
|
|
156
|
+
eac_fs (~> 0.4)
|
|
157
|
+
eac_ruby_utils (~> 0.68)
|
|
158
|
+
avm-eac_ruby_base1 (0.1.0)
|
|
159
|
+
avm (~> 0.6)
|
|
160
|
+
eac_ruby_utils (~> 0.76)
|
|
161
|
+
avm-eac_ubuntu_base0 (0.2.0)
|
|
162
|
+
avm (~> 0.7)
|
|
163
|
+
eac_ruby_utils (~> 0.77, >= 0.77.1)
|
|
164
|
+
eac_templates (~> 0.2)
|
|
147
165
|
avm-eac_webapp_base0 (0.1.1)
|
|
148
166
|
avm (~> 0.1)
|
|
149
167
|
eac_ruby_utils (~> 0.68)
|
|
150
|
-
avm-
|
|
168
|
+
avm-files (0.2.0)
|
|
169
|
+
avm (~> 0.6)
|
|
170
|
+
eac_fs (~> 0.5)
|
|
171
|
+
eac_ruby_utils (~> 0.76)
|
|
172
|
+
eac_templates (~> 0.2)
|
|
173
|
+
avm-git (0.1.0)
|
|
174
|
+
avm-files (~> 0.1)
|
|
175
|
+
eac_git (~> 0.9)
|
|
176
|
+
eac_ruby_utils (~> 0.76)
|
|
177
|
+
avm-tools (0.111.0)
|
|
151
178
|
aranha-parsers (~> 0.4)
|
|
152
179
|
avm (~> 0.1)
|
|
153
180
|
avm-eac_asciidoctor_base0
|
|
181
|
+
avm-eac_redmine_base0 (~> 0.3)
|
|
182
|
+
avm-eac_ruby_base1 (~> 0.1)
|
|
183
|
+
avm-eac_ubuntu_base0 (~> 0.2)
|
|
184
|
+
avm-files (~> 0.1)
|
|
185
|
+
avm-git (~> 0.1)
|
|
154
186
|
clipboard (~> 1.3, >= 1.3.3)
|
|
155
187
|
curb (~> 0.9.10)
|
|
156
188
|
eac_ruby_base0 (~> 0.7, >= 0.7.2)
|
|
@@ -174,7 +206,7 @@ GEM
|
|
|
174
206
|
thread_safe (~> 0.3, >= 0.3.1)
|
|
175
207
|
diff-lcs (1.4.4)
|
|
176
208
|
dry-inflector (0.2.1)
|
|
177
|
-
eac_git (0.
|
|
209
|
+
eac_git (0.9.0)
|
|
178
210
|
eac_ruby_utils (~> 0.74)
|
|
179
211
|
parseconfig (~> 1.0, >= 1.0.8)
|
|
180
212
|
eac_ruby_gem_support (0.4.0)
|
|
@@ -202,12 +234,14 @@ GEM
|
|
|
202
234
|
faraday-net_http_persistent (1.2.0)
|
|
203
235
|
faraday-patron (1.0.0)
|
|
204
236
|
faraday-rack (1.0.0)
|
|
237
|
+
faraday_middleware (1.2.0)
|
|
238
|
+
faraday (~> 1.0)
|
|
205
239
|
filesize (0.2.0)
|
|
206
240
|
git (1.9.1)
|
|
207
241
|
rchardet (~> 1.8)
|
|
208
242
|
hpricot (0.8.6)
|
|
209
243
|
htmlbeautifier (1.3.1)
|
|
210
|
-
i18n (1.8.
|
|
244
|
+
i18n (1.8.11)
|
|
211
245
|
concurrent-ruby (~> 1.0)
|
|
212
246
|
ice_nine (0.11.2)
|
|
213
247
|
inifile (3.0.0)
|
|
@@ -220,14 +254,14 @@ GEM
|
|
|
220
254
|
racc (~> 1.4)
|
|
221
255
|
ofx-parser (1.1.0)
|
|
222
256
|
hpricot (>= 0.6)
|
|
223
|
-
os (1.1.
|
|
257
|
+
os (1.1.4)
|
|
224
258
|
parallel (1.21.0)
|
|
225
259
|
parseconfig (1.1.2)
|
|
226
260
|
parser (3.0.2.0)
|
|
227
261
|
ast (~> 2.4.1)
|
|
228
262
|
parslet (1.8.2)
|
|
229
263
|
public_suffix (4.0.6)
|
|
230
|
-
racc (1.
|
|
264
|
+
racc (1.6.0)
|
|
231
265
|
rack (2.2.3)
|
|
232
266
|
rainbow (3.0.0)
|
|
233
267
|
rchardet (1.8.0)
|
|
@@ -245,7 +279,7 @@ GEM
|
|
|
245
279
|
rspec-mocks (3.10.2)
|
|
246
280
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
247
281
|
rspec-support (~> 3.10.0)
|
|
248
|
-
rspec-support (3.10.
|
|
282
|
+
rspec-support (3.10.3)
|
|
249
283
|
rubocop (0.80.1)
|
|
250
284
|
jaro_winkler (~> 1.5.1)
|
|
251
285
|
parallel (~> 1.10)
|
|
@@ -279,7 +313,7 @@ GEM
|
|
|
279
313
|
axiom-types (~> 0.1)
|
|
280
314
|
coercible (~> 1.0)
|
|
281
315
|
descendants_tracker (~> 0.0, >= 0.0.3)
|
|
282
|
-
zeitwerk (2.
|
|
316
|
+
zeitwerk (2.5.1)
|
|
283
317
|
|
|
284
318
|
PLATFORMS
|
|
285
319
|
x86_64-linux
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'eac_cli/core_ext'
|
|
4
|
-
require '
|
|
4
|
+
require 'eac_fs/traversable'
|
|
5
5
|
require 'ehbrs/vg/wii/file_move'
|
|
6
6
|
require 'ehbrs/vg/wii/game_file'
|
|
7
7
|
|
|
@@ -10,7 +10,7 @@ module Ehbrs
|
|
|
10
10
|
class Runner
|
|
11
11
|
class Vg
|
|
12
12
|
class Wii
|
|
13
|
-
include ::
|
|
13
|
+
include ::EacFs::Traversable
|
|
14
14
|
|
|
15
15
|
runner_with :help do
|
|
16
16
|
desc 'Manipulação de imagens de jogo Wii.'
|
data/lib/ehbrs/tools/version.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'eac_fs/traversable'
|
|
4
4
|
require 'eac_ruby_utils/core_ext'
|
|
5
5
|
require 'ehbrs/videos/unsupported/file'
|
|
6
6
|
|
|
@@ -8,7 +8,7 @@ module Ehbrs
|
|
|
8
8
|
module Videos
|
|
9
9
|
module Unsupported
|
|
10
10
|
class Search
|
|
11
|
-
include ::
|
|
11
|
+
include ::EacFs::Traversable
|
|
12
12
|
enable_speaker
|
|
13
13
|
enable_simple_cache
|
|
14
14
|
|
|
@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
|
|
|
16
16
|
s.add_dependency 'addressable', '~> 2.7'
|
|
17
17
|
s.add_dependency 'curb', '~> 0.9.10'
|
|
18
18
|
s.add_dependency 'eac_ruby_utils', '~> 0.74'
|
|
19
|
+
s.add_dependency 'faraday_middleware'
|
|
19
20
|
s.add_dependency 'httpclient', '~> 2.8', '>= 2.8.3'
|
|
20
21
|
s.add_dependency 'nokogiri', '~> 1.12', '>= 1.12.4'
|
|
21
22
|
s.add_dependency 'ofx-parser', '~> 1.1.0'
|
|
@@ -11,7 +11,9 @@ module Aranha
|
|
|
11
11
|
class << self
|
|
12
12
|
def from_content(content)
|
|
13
13
|
::EacRubyUtils::Fs::Temp.on_file do |path|
|
|
14
|
-
path.
|
|
14
|
+
::File.open(path.to_s, 'w:UTF-8') do |f|
|
|
15
|
+
f.write content.force_encoding('UTF-8')
|
|
16
|
+
end
|
|
15
17
|
r = new(path.to_path)
|
|
16
18
|
r.content
|
|
17
19
|
r
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aranha/parsers/html/node/base'
|
|
4
|
+
require 'eac_ruby_utils/core_ext'
|
|
5
|
+
|
|
6
|
+
module Aranha
|
|
7
|
+
module Parsers
|
|
8
|
+
module Html
|
|
9
|
+
module Node
|
|
10
|
+
class Default < ::Aranha::Parsers::Html::Node::Base
|
|
11
|
+
module NumericSupport
|
|
12
|
+
def integer_value(node, xpath)
|
|
13
|
+
r = string_value(node, xpath)
|
|
14
|
+
return nil if r.blank?
|
|
15
|
+
|
|
16
|
+
m = /\d+/.match(r)
|
|
17
|
+
raise "Integer not found in \"#{r}\"" unless m
|
|
18
|
+
|
|
19
|
+
m[0].to_i
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def integer_optional_value(node, xpath)
|
|
23
|
+
r = string_value(node, xpath)
|
|
24
|
+
m = /\d+/.match(r)
|
|
25
|
+
m ? m[0].to_i : nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def float_value(node, xpath)
|
|
29
|
+
parse_float(node, xpath, true)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def float_optional_value(node, xpath)
|
|
33
|
+
parse_float(node, xpath, false)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def us_decimal_value(node, xpath)
|
|
37
|
+
parse_us_decimal(node, xpath, true)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def us_decimal_optional_value(node, xpath)
|
|
41
|
+
parse_us_decimal(node, xpath, false)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def parse_float(node, xpath, required)
|
|
47
|
+
s = string_value(node, xpath)
|
|
48
|
+
m = /\d+(?:[\.\,](\d+))?/.match(s)
|
|
49
|
+
if m
|
|
50
|
+
m[0].delete('.').tr(',', '.').to_f
|
|
51
|
+
elsif required
|
|
52
|
+
raise "Float value not found in \"#{s}\""
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def parse_us_decimal(node, xpath, required)
|
|
57
|
+
s = string_value(node, xpath)
|
|
58
|
+
m = /\d+(?:[\.\,](\d+))?/.match(s)
|
|
59
|
+
if m
|
|
60
|
+
m[0].delete(',').to_f
|
|
61
|
+
elsif required
|
|
62
|
+
raise "US decimal value not found in \"#{s}\""
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aranha/parsers/html/node/base'
|
|
4
|
+
require 'eac_ruby_utils/core_ext'
|
|
5
|
+
|
|
6
|
+
module Aranha
|
|
7
|
+
module Parsers
|
|
8
|
+
module Html
|
|
9
|
+
module Node
|
|
10
|
+
class Default < ::Aranha::Parsers::Html::Node::Base
|
|
11
|
+
module StringSupport
|
|
12
|
+
def quoted_value(node, xpath)
|
|
13
|
+
s = string_value(node, xpath)
|
|
14
|
+
return '' unless s
|
|
15
|
+
|
|
16
|
+
m = /\"([^\"]+)\"/.match(s)
|
|
17
|
+
return m[1] if m
|
|
18
|
+
|
|
19
|
+
''
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def regxep(node, xpath, pattern)
|
|
23
|
+
s = string_value(node, xpath)
|
|
24
|
+
m = pattern.match(s)
|
|
25
|
+
return m if m
|
|
26
|
+
|
|
27
|
+
raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def string_value(node, xpath)
|
|
31
|
+
if node.at_xpath(xpath)
|
|
32
|
+
sanitize_string(node.at_xpath(xpath).text)
|
|
33
|
+
else
|
|
34
|
+
''
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def string_recursive_value(node, xpath, required = true)
|
|
39
|
+
root = node.at_xpath(xpath)
|
|
40
|
+
if root.blank?
|
|
41
|
+
return nil unless required
|
|
42
|
+
|
|
43
|
+
raise "No node found (Xpath: #{xpath})"
|
|
44
|
+
end
|
|
45
|
+
result = string_recursive(root)
|
|
46
|
+
return result if result.present?
|
|
47
|
+
return nil unless required
|
|
48
|
+
|
|
49
|
+
raise "String blank (Xpath: #{xpath})"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def string_recursive_optional_value(node, xpath)
|
|
53
|
+
string_recursive_value(node, xpath, false)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def sanitize_string(obj)
|
|
59
|
+
obj.to_s.tr("\u00A0", ' ').strip
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def string_recursive(node)
|
|
63
|
+
return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
|
|
64
|
+
|
|
65
|
+
s = ''
|
|
66
|
+
node.children.each do |child|
|
|
67
|
+
child_s = string_recursive(child)
|
|
68
|
+
s += ' ' + child_s if child_s.present?
|
|
69
|
+
end
|
|
70
|
+
sanitize_string(s)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -1,71 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aranha/parsers/html/node/base'
|
|
4
|
+
require 'eac_ruby_utils/core_ext'
|
|
4
5
|
|
|
5
6
|
module Aranha
|
|
6
7
|
module Parsers
|
|
7
8
|
module Html
|
|
8
9
|
module Node
|
|
9
10
|
class Default < ::Aranha::Parsers::Html::Node::Base
|
|
10
|
-
|
|
11
|
-
if node.at_xpath(xpath)
|
|
12
|
-
sanitize_string(node.at_xpath(xpath).text)
|
|
13
|
-
else
|
|
14
|
-
''
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def string_recursive_value(node, xpath, required = true)
|
|
19
|
-
root = node.at_xpath(xpath)
|
|
20
|
-
if root.blank?
|
|
21
|
-
return nil unless required
|
|
22
|
-
|
|
23
|
-
raise "No node found (Xpath: #{xpath})"
|
|
24
|
-
end
|
|
25
|
-
result = string_recursive(root)
|
|
26
|
-
return result if result.present?
|
|
27
|
-
return nil unless required
|
|
28
|
-
|
|
29
|
-
raise "String blank (Xpath: #{xpath})"
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def string_recursive_optional_value(node, xpath)
|
|
33
|
-
string_recursive_value(node, xpath, false)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def quoted_value(node, xpath)
|
|
37
|
-
s = string_value(node, xpath)
|
|
38
|
-
return '' unless s
|
|
39
|
-
|
|
40
|
-
m = /\"([^\"]+)\"/.match(s)
|
|
41
|
-
return m[1] if m
|
|
42
|
-
|
|
43
|
-
''
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def integer_value(node, xpath)
|
|
47
|
-
r = string_value(node, xpath)
|
|
48
|
-
return nil if r.blank?
|
|
49
|
-
|
|
50
|
-
m = /\d+/.match(r)
|
|
51
|
-
raise "Integer not found in \"#{r}\"" unless m
|
|
52
|
-
|
|
53
|
-
m[0].to_i
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
def integer_optional_value(node, xpath)
|
|
57
|
-
r = string_value(node, xpath)
|
|
58
|
-
m = /\d+/.match(r)
|
|
59
|
-
m ? m[0].to_i : nil
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def float_value(node, xpath)
|
|
63
|
-
parse_float(node, xpath, true)
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def float_optional_value(node, xpath)
|
|
67
|
-
parse_float(node, xpath, false)
|
|
68
|
-
end
|
|
11
|
+
require_sub __FILE__, include_modules: true
|
|
69
12
|
|
|
70
13
|
def array_value(node, xpath)
|
|
71
14
|
r = node.xpath(xpath).map { |n| n.text.strip }
|
|
@@ -84,41 +27,6 @@ module Aranha
|
|
|
84
27
|
m = /(\d+) m/.match(join_value(node, xpath))
|
|
85
28
|
m ? m[1].to_i : nil
|
|
86
29
|
end
|
|
87
|
-
|
|
88
|
-
def regxep(node, xpath, pattern)
|
|
89
|
-
s = string_value(node, xpath)
|
|
90
|
-
m = pattern.match(s)
|
|
91
|
-
return m if m
|
|
92
|
-
|
|
93
|
-
raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
private
|
|
97
|
-
|
|
98
|
-
def parse_float(node, xpath, required)
|
|
99
|
-
s = string_value(node, xpath)
|
|
100
|
-
m = /\d+(?:[\.\,](\d+))?/.match(s)
|
|
101
|
-
if m
|
|
102
|
-
m[0].delete('.').tr(',', '.').to_f
|
|
103
|
-
elsif required
|
|
104
|
-
raise "Float value not found in \"#{s}\""
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def sanitize_string(obj)
|
|
109
|
-
obj.to_s.tr("\u00A0", ' ').strip
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
def string_recursive(node)
|
|
113
|
-
return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
|
|
114
|
-
|
|
115
|
-
s = ''
|
|
116
|
-
node.children.each do |child|
|
|
117
|
-
child_s = string_recursive(child)
|
|
118
|
-
s += ' ' + child_s if child_s.present?
|
|
119
|
-
end
|
|
120
|
-
sanitize_string(s)
|
|
121
|
-
end
|
|
122
30
|
end
|
|
123
31
|
end
|
|
124
32
|
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aranha/parsers/base'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Aranha
|
|
7
|
+
module Parsers
|
|
8
|
+
module Json
|
|
9
|
+
class Base < ::Aranha::Parsers::Base
|
|
10
|
+
def data
|
|
11
|
+
default_data
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def default_data
|
|
15
|
+
::JSON.parse(content)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'addressable'
|
|
4
4
|
require 'curb'
|
|
5
5
|
require 'aranha/parsers/source_address/fetch_content_error'
|
|
6
|
+
require 'faraday_middleware'
|
|
6
7
|
|
|
7
8
|
module Aranha
|
|
8
9
|
module Parsers
|
|
@@ -32,11 +33,18 @@ module Aranha
|
|
|
32
33
|
source
|
|
33
34
|
end
|
|
34
35
|
|
|
36
|
+
def final_url
|
|
37
|
+
content unless @final_url
|
|
38
|
+
@final_url
|
|
39
|
+
end
|
|
40
|
+
|
|
35
41
|
def content
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
42
|
+
conn = ::Faraday.new do |f|
|
|
43
|
+
f.request :retry # retry transient failures
|
|
44
|
+
f.response :follow_redirects # follow redirects
|
|
45
|
+
end
|
|
46
|
+
c = conn.get(url)
|
|
47
|
+
return c.body if c.status == 200
|
|
40
48
|
|
|
41
49
|
raise ::Aranha::Parsers::SourceAddress::FetchContentError,
|
|
42
50
|
"Get #{url} returned #{c.status.to_i}"
|
|
@@ -45,17 +53,6 @@ module Aranha
|
|
|
45
53
|
def serialize
|
|
46
54
|
url
|
|
47
55
|
end
|
|
48
|
-
|
|
49
|
-
private
|
|
50
|
-
|
|
51
|
-
def curl_perform(curl)
|
|
52
|
-
unless curl.perform
|
|
53
|
-
raise(::Aranha::Parsers::SourceAddress::FetchContentError,
|
|
54
|
-
"Curl perform failed (URL: #{url})")
|
|
55
|
-
end
|
|
56
|
-
rescue Curl::Err::CurlError => e
|
|
57
|
-
raise ::Aranha::Parsers::SourceAddress::FetchContentError, "CURL error: #{e.class.name}"
|
|
58
|
-
end
|
|
59
56
|
end
|
|
60
57
|
end
|
|
61
58
|
end
|
|
@@ -5,10 +5,12 @@ require 'eac_fs/patches/module/fs_cache'
|
|
|
5
5
|
class Object
|
|
6
6
|
# @return [EacFs::Cache]
|
|
7
7
|
def fs_cache
|
|
8
|
-
|
|
8
|
+
oid = fs_cache_object_id
|
|
9
|
+
oid = [oid.to_s] unless oid.is_a?(::Enumerable)
|
|
10
|
+
oid.inject(self.class.fs_cache) { |a, e| a.child(e.to_s) }
|
|
9
11
|
end
|
|
10
12
|
|
|
11
|
-
# @return [String]
|
|
13
|
+
# @return [String, Array<String>]
|
|
12
14
|
def fs_cache_object_id
|
|
13
15
|
raise 'Abstract method hit'
|
|
14
16
|
end
|