yasuri 2.0.11 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ruby.yml +35 -0
- data/.gitignore +1 -2
- data/.ruby-version +1 -0
- data/.travis.yml +1 -3
- data/README.md +88 -19
- data/USAGE.ja.md +325 -63
- data/USAGE.md +335 -69
- data/exe/yasuri +5 -0
- data/lib/yasuri.rb +1 -0
- data/lib/yasuri/version.rb +1 -1
- data/lib/yasuri/yasuri.rb +80 -39
- data/lib/yasuri/yasuri_cli.rb +64 -0
- data/lib/yasuri/yasuri_links_node.rb +10 -6
- data/lib/yasuri/yasuri_map_node.rb +39 -0
- data/lib/yasuri/yasuri_node.rb +24 -3
- data/lib/yasuri/yasuri_node_generator.rb +16 -11
- data/lib/yasuri/yasuri_paginate_node.rb +18 -6
- data/lib/yasuri/yasuri_struct_node.rb +8 -4
- data/lib/yasuri/yasuri_text_node.rb +11 -4
- data/spec/cli_resources/tree.json +8 -0
- data/spec/cli_resources/tree.yml +5 -0
- data/spec/cli_resources/tree_wrong.json +9 -0
- data/spec/cli_resources/tree_wrong.yml +6 -0
- data/spec/htdocs/struct/structual_links.html +30 -0
- data/spec/htdocs/{structual_text.html → struct/structual_text.html} +0 -0
- data/spec/spec_helper.rb +1 -6
- data/spec/yasuri_cli_spec.rb +83 -0
- data/spec/yasuri_links_node_spec.rb +12 -4
- data/spec/yasuri_map_spec.rb +76 -0
- data/spec/yasuri_paginate_node_spec.rb +43 -0
- data/spec/yasuri_spec.rb +199 -84
- data/spec/yasuri_struct_node_spec.rb +42 -1
- data/yasuri.gemspec +5 -3
- metadata +52 -19
@@ -12,7 +12,7 @@ describe 'Yasuri' do
|
|
12
12
|
describe '::StructNode' do
|
13
13
|
before do
|
14
14
|
@agent = Mechanize.new
|
15
|
-
@page = @agent.get(uri + "/structual_text.html")
|
15
|
+
@page = @agent.get(uri + "/struct/structual_text.html")
|
16
16
|
|
17
17
|
@table_1996 = [
|
18
18
|
{ "title" => "The Perfect Insider",
|
@@ -132,4 +132,45 @@ describe 'Yasuri' do
|
|
132
132
|
end
|
133
133
|
|
134
134
|
end
|
135
|
+
|
136
|
+
describe '::StructNode::Links' do
|
137
|
+
before do
|
138
|
+
@agent = Mechanize.new
|
139
|
+
@page = @agent.get(uri + "/struct/structual_links.html")
|
140
|
+
|
141
|
+
@table = [
|
142
|
+
{ "title" => "Child01,02",
|
143
|
+
"child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
|
144
|
+
|
145
|
+
{ "title" => "Child01,02,03",
|
146
|
+
"child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
|
147
|
+
]
|
148
|
+
end
|
149
|
+
|
150
|
+
it 'return child node in links inside struct' do
|
151
|
+
node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
|
152
|
+
Yasuri::TextNode.new('./td[1]', "title"),
|
153
|
+
Yasuri::LinksNode.new('./td/a', "child", [
|
154
|
+
Yasuri::TextNode.new('/html/body/p', "p"),
|
155
|
+
])
|
156
|
+
])
|
157
|
+
expected = @table
|
158
|
+
actual = node.inject(@agent, @page)
|
159
|
+
expect(actual).to match expected
|
160
|
+
end
|
161
|
+
end # descrive
|
162
|
+
|
163
|
+
describe '::StructNode::Pages' do
|
164
|
+
before do
|
165
|
+
@agent = Mechanize.new
|
166
|
+
@page = @agent.get(uri + "/struct/structual_text.html") #dummy
|
167
|
+
end
|
168
|
+
|
169
|
+
it 'not supported' do
|
170
|
+
node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
|
171
|
+
Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
|
172
|
+
])
|
173
|
+
expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
|
174
|
+
end
|
175
|
+
end
|
135
176
|
end
|
data/yasuri.gemspec
CHANGED
@@ -14,12 +14,13 @@ Gem::Specification.new do |spec|
|
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
17
|
-
spec.executables = spec.files.grep(%r{^
|
17
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
18
|
+
spec.bindir = "exe"
|
18
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
20
|
spec.require_paths = ["lib"]
|
20
21
|
|
21
|
-
spec.add_development_dependency "bundler"
|
22
|
-
spec.add_development_dependency "rake"
|
22
|
+
spec.add_development_dependency "bundler"
|
23
|
+
spec.add_development_dependency "rake"
|
23
24
|
spec.add_development_dependency "rspec"
|
24
25
|
spec.add_development_dependency "fuubar"
|
25
26
|
spec.add_development_dependency "glint"
|
@@ -28,4 +29,5 @@ Gem::Specification.new do |spec|
|
|
28
29
|
spec.add_development_dependency "codeclimate-test-reporter"
|
29
30
|
|
30
31
|
spec.add_dependency "mechanize"
|
32
|
+
spec.add_dependency "thor"
|
31
33
|
end
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yasuri
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAC
|
8
|
-
autorequire:
|
9
|
-
bindir:
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '0'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,16 +136,33 @@ dependencies:
|
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: thor
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
139
153
|
description: Yasuri is an easy web-scraping library for supporting "Mechanize".
|
140
154
|
email:
|
141
155
|
- tac@tac42.net
|
142
|
-
executables:
|
156
|
+
executables:
|
157
|
+
- yasuri
|
143
158
|
extensions: []
|
144
159
|
extra_rdoc_files: []
|
145
160
|
files:
|
146
161
|
- ".coveralls.yml"
|
162
|
+
- ".github/workflows/ruby.yml"
|
147
163
|
- ".gitignore"
|
148
164
|
- ".rspec"
|
165
|
+
- ".ruby-version"
|
149
166
|
- ".travis.yml"
|
150
167
|
- Gemfile
|
151
168
|
- LICENSE
|
@@ -154,15 +171,22 @@ files:
|
|
154
171
|
- USAGE.ja.md
|
155
172
|
- USAGE.md
|
156
173
|
- app.rb
|
174
|
+
- exe/yasuri
|
157
175
|
- lib/yasuri.rb
|
158
176
|
- lib/yasuri/version.rb
|
159
177
|
- lib/yasuri/yasuri.rb
|
178
|
+
- lib/yasuri/yasuri_cli.rb
|
160
179
|
- lib/yasuri/yasuri_links_node.rb
|
180
|
+
- lib/yasuri/yasuri_map_node.rb
|
161
181
|
- lib/yasuri/yasuri_node.rb
|
162
182
|
- lib/yasuri/yasuri_node_generator.rb
|
163
183
|
- lib/yasuri/yasuri_paginate_node.rb
|
164
184
|
- lib/yasuri/yasuri_struct_node.rb
|
165
185
|
- lib/yasuri/yasuri_text_node.rb
|
186
|
+
- spec/cli_resources/tree.json
|
187
|
+
- spec/cli_resources/tree.yml
|
188
|
+
- spec/cli_resources/tree_wrong.json
|
189
|
+
- spec/cli_resources/tree_wrong.yml
|
166
190
|
- spec/htdocs/child01.html
|
167
191
|
- spec/htdocs/child01_sub.html
|
168
192
|
- spec/htdocs/child02.html
|
@@ -174,10 +198,13 @@ files:
|
|
174
198
|
- spec/htdocs/pagination/page02.html
|
175
199
|
- spec/htdocs/pagination/page03.html
|
176
200
|
- spec/htdocs/pagination/page04.html
|
177
|
-
- spec/htdocs/
|
201
|
+
- spec/htdocs/struct/structual_links.html
|
202
|
+
- spec/htdocs/struct/structual_text.html
|
178
203
|
- spec/servers/httpserver.rb
|
179
204
|
- spec/spec_helper.rb
|
205
|
+
- spec/yasuri_cli_spec.rb
|
180
206
|
- spec/yasuri_links_node_spec.rb
|
207
|
+
- spec/yasuri_map_spec.rb
|
181
208
|
- spec/yasuri_node_spec.rb
|
182
209
|
- spec/yasuri_paginate_node_spec.rb
|
183
210
|
- spec/yasuri_spec.rb
|
@@ -188,7 +215,7 @@ homepage: https://github.com/tac0x2a/yasuri
|
|
188
215
|
licenses:
|
189
216
|
- MIT
|
190
217
|
metadata: {}
|
191
|
-
post_install_message:
|
218
|
+
post_install_message:
|
192
219
|
rdoc_options: []
|
193
220
|
require_paths:
|
194
221
|
- lib
|
@@ -203,12 +230,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
203
230
|
- !ruby/object:Gem::Version
|
204
231
|
version: '0'
|
205
232
|
requirements: []
|
206
|
-
|
207
|
-
|
208
|
-
signing_key:
|
233
|
+
rubygems_version: 3.2.3
|
234
|
+
signing_key:
|
209
235
|
specification_version: 4
|
210
236
|
summary: Yasuri is easy scraping library.
|
211
237
|
test_files:
|
238
|
+
- spec/cli_resources/tree.json
|
239
|
+
- spec/cli_resources/tree.yml
|
240
|
+
- spec/cli_resources/tree_wrong.json
|
241
|
+
- spec/cli_resources/tree_wrong.yml
|
212
242
|
- spec/htdocs/child01.html
|
213
243
|
- spec/htdocs/child01_sub.html
|
214
244
|
- spec/htdocs/child02.html
|
@@ -220,10 +250,13 @@ test_files:
|
|
220
250
|
- spec/htdocs/pagination/page02.html
|
221
251
|
- spec/htdocs/pagination/page03.html
|
222
252
|
- spec/htdocs/pagination/page04.html
|
223
|
-
- spec/htdocs/
|
253
|
+
- spec/htdocs/struct/structual_links.html
|
254
|
+
- spec/htdocs/struct/structual_text.html
|
224
255
|
- spec/servers/httpserver.rb
|
225
256
|
- spec/spec_helper.rb
|
257
|
+
- spec/yasuri_cli_spec.rb
|
226
258
|
- spec/yasuri_links_node_spec.rb
|
259
|
+
- spec/yasuri_map_spec.rb
|
227
260
|
- spec/yasuri_node_spec.rb
|
228
261
|
- spec/yasuri_paginate_node_spec.rb
|
229
262
|
- spec/yasuri_spec.rb
|