tsumamigui 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +109 -0
- data/.hound.yml +2 -0
- data/.rubocop.yml +3 -1
- data/CHANGELOG.md +10 -0
- data/README.md +1 -1
- data/lib/tsumamigui/parser.rb +4 -8
- data/lib/tsumamigui/request.rb +5 -10
- data/lib/tsumamigui/version.rb +1 -1
- metadata +4 -3
- data/circle.yml +0 -45
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: beb1d6f4f9c9c2df335b58c81ae398a4428845fa
|
|
4
|
+
data.tar.gz: 2ae6515d29e64f28cb8e7f58c652d2cb61b1c278
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc852aee3ae850519d9de601a158c5158f3f7574d34cee4ef39e563a0604a403463d5d34b072350084ddb93ef156fe7df8d9024cec730fc6f2c9f56680515ac9
|
|
7
|
+
data.tar.gz: 236839d9eeadd3c4da5f416a34debf19361e042f50abdc2da525ea42c56c787312aa586ec24837aba153be366c8aad0fcffed216f6f2845b1482dfb2feb1041a
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
|
|
2
|
+
defaults: &defaults
|
|
3
|
+
working_directory: ~/tsumamigui
|
|
4
|
+
environment:
|
|
5
|
+
CC_TEST_REPORTER_ID: fb2b65ee179ec4ba3ac0f60d2047823566cc14dac0578c4c401532e682f5d435
|
|
6
|
+
|
|
7
|
+
jobs: &jobs
|
|
8
|
+
jobs:
|
|
9
|
+
- build
|
|
10
|
+
- lint:
|
|
11
|
+
requires:
|
|
12
|
+
- build
|
|
13
|
+
- rspec:
|
|
14
|
+
requires:
|
|
15
|
+
- build
|
|
16
|
+
- rspec-2_3_5
|
|
17
|
+
- rspec-2_2_8
|
|
18
|
+
|
|
19
|
+
version: 2
|
|
20
|
+
jobs:
|
|
21
|
+
build:
|
|
22
|
+
<<: *defaults
|
|
23
|
+
docker:
|
|
24
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
25
|
+
steps:
|
|
26
|
+
- checkout
|
|
27
|
+
- restore_cache:
|
|
28
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
29
|
+
- run: bundle install --path vendor/bundle
|
|
30
|
+
- save_cache:
|
|
31
|
+
paths:
|
|
32
|
+
- vendor/bundle
|
|
33
|
+
- Gemfile.lock
|
|
34
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
35
|
+
|
|
36
|
+
lint:
|
|
37
|
+
docker:
|
|
38
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
39
|
+
steps:
|
|
40
|
+
- checkout
|
|
41
|
+
- restore_cache:
|
|
42
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
43
|
+
- run: bundle install --path vendor/bundle
|
|
44
|
+
- run: bundle exec rubocop
|
|
45
|
+
|
|
46
|
+
rspec:
|
|
47
|
+
<<: *defaults
|
|
48
|
+
docker:
|
|
49
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
50
|
+
steps:
|
|
51
|
+
- checkout
|
|
52
|
+
- restore_cache:
|
|
53
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
54
|
+
|
|
55
|
+
- run: curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
|
56
|
+
- run: chmod +x ./cc-test-reporter
|
|
57
|
+
- run: bundle install --path vendor/bundle
|
|
58
|
+
- run: bundle exec rspec
|
|
59
|
+
- run: ./cc-test-reporter after-build -t simplecov --exit-code $?
|
|
60
|
+
- store_artifacts:
|
|
61
|
+
path: coverage
|
|
62
|
+
|
|
63
|
+
rspec-2_3_5:
|
|
64
|
+
<<: *defaults
|
|
65
|
+
docker:
|
|
66
|
+
- image: circleci/ruby:2.3.5-node-browsers
|
|
67
|
+
steps:
|
|
68
|
+
- checkout
|
|
69
|
+
- restore_cache:
|
|
70
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_3_5
|
|
71
|
+
- run: bundle install --path vendor/bundle
|
|
72
|
+
- run: bundle exec rspec
|
|
73
|
+
- save_cache:
|
|
74
|
+
paths:
|
|
75
|
+
- vendor/bundle
|
|
76
|
+
- Gemfile.lock
|
|
77
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_3_5
|
|
78
|
+
|
|
79
|
+
rspec-2_2_8:
|
|
80
|
+
<<: *defaults
|
|
81
|
+
docker:
|
|
82
|
+
- image: circleci/ruby:2.2.8-node-browsers
|
|
83
|
+
steps:
|
|
84
|
+
- checkout
|
|
85
|
+
- restore_cache:
|
|
86
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_2_8
|
|
87
|
+
- run: bundle install --path vendor/bundle
|
|
88
|
+
- run: bundle exec rspec
|
|
89
|
+
- save_cache:
|
|
90
|
+
paths:
|
|
91
|
+
- vendor/bundle
|
|
92
|
+
- Gemfile.lock
|
|
93
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_2_8
|
|
94
|
+
|
|
95
|
+
workflows:
|
|
96
|
+
version: 2
|
|
97
|
+
build_and_test:
|
|
98
|
+
<<: *jobs
|
|
99
|
+
|
|
100
|
+
scheduled-build_and_test:
|
|
101
|
+
triggers:
|
|
102
|
+
- schedule:
|
|
103
|
+
cron: "0 23 * * *"
|
|
104
|
+
filters:
|
|
105
|
+
branches:
|
|
106
|
+
only:
|
|
107
|
+
- master
|
|
108
|
+
<<: *jobs
|
|
109
|
+
|
data/.hound.yml
ADDED
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Tsumamigui
|
|
2
|
-
[](https://badge.fury.io/rb/tsumamigui) [](https://circleci.com/gh/obiyuta/tsumamigui/tree/master) [](https://codeclimate.com/github/obiyuta/tsumamigui) [](https://codeclimate.com/github/obiyuta/tsumamigui/coverage) [](https://gemnasium.com/github.com/obiyuta/tsumamigui) [](http://inch-ci.org/github/obiyuta/tsumamigui)
|
|
2
|
+
[](https://badge.fury.io/rb/tsumamigui) [](https://circleci.com/gh/obiyuta/tsumamigui/tree/master) [](https://codeclimate.com/github/obiyuta/tsumamigui) [](https://codeclimate.com/github/obiyuta/tsumamigui/coverage) [](https://gemnasium.com/github.com/obiyuta/tsumamigui) [](http://inch-ci.org/github/obiyuta/tsumamigui) [](https://codebeat.co/projects/github-com-obiyuta-tsumamigui-master)
|
|
3
3
|
|
|
4
4
|
Tsumamigui（つまみぐい） is a simple and hussle-free Ruby web scraping library.
|
|
5
5
|
|
data/lib/tsumamigui/parser.rb
CHANGED
|
@@ -27,14 +27,12 @@ module Tsumamigui
|
|
|
27
27
|
# @return [Array<Hash>] parsed responses
|
|
28
28
|
# @raise [Tsumamigui::ParserError]
|
|
29
29
|
def parse(responses)
|
|
30
|
-
|
|
31
|
-
responses.each do |res|
|
|
30
|
+
responses.map do |res|
|
|
32
31
|
url, html, charset = res.to_array
|
|
33
32
|
result = extract(Nokogiri::HTML.parse(html, nil, charset))
|
|
34
33
|
result[:scraped_from] = url
|
|
35
|
-
|
|
34
|
+
result
|
|
36
35
|
end
|
|
37
|
-
results
|
|
38
36
|
rescue => e
|
|
39
37
|
raise ParserError, e.message
|
|
40
38
|
end
|
|
@@ -44,11 +42,9 @@ module Tsumamigui
|
|
|
44
42
|
# @return [Hash] xpath and its key
|
|
45
43
|
# @raise [Tsumamigui::ParserError]
|
|
46
44
|
def extract(document)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
result[k] = document.xpath(v).to_s
|
|
45
|
+
@xpath.each_with_object({}) do |(key, value), hash|
|
|
46
|
+
hash[key] = document.xpath(value).to_s
|
|
50
47
|
end
|
|
51
|
-
result
|
|
52
48
|
rescue => e
|
|
53
49
|
raise ParserError, e.message
|
|
54
50
|
end
|
data/lib/tsumamigui/request.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Tsumamigui
|
|
|
9
9
|
# @param [Array<String>] urls
|
|
10
10
|
# @return [Array<Tsumamigui::Response>] response instances
|
|
11
11
|
def run(*urls)
|
|
12
|
-
new(urls).
|
|
12
|
+
new(urls).exec_request
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -22,21 +22,16 @@ module Tsumamigui
|
|
|
22
22
|
def initialize(*urls)
|
|
23
23
|
@urls = urls.flatten
|
|
24
24
|
raise ArgumentError, 'No argument is specified' if @urls.empty?
|
|
25
|
-
@urls
|
|
26
25
|
end
|
|
27
26
|
|
|
28
|
-
private
|
|
29
|
-
|
|
30
27
|
# Execute request and return responses
|
|
31
28
|
# @return [Array<Tsumamigui::Response>] response instances
|
|
32
|
-
def
|
|
33
|
-
|
|
34
|
-
@urls.each do |url|
|
|
35
|
-
results.push(fetch(url))
|
|
36
|
-
end
|
|
37
|
-
results
|
|
29
|
+
def exec_request
|
|
30
|
+
@urls.map { |url| fetch(url) }
|
|
38
31
|
end
|
|
39
32
|
|
|
33
|
+
private
|
|
34
|
+
|
|
40
35
|
# Fetch HTML from url and convert to response object
|
|
41
36
|
# @param [String] url
|
|
42
37
|
# @return [Tsumamigui::Response] response
|
data/lib/tsumamigui/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tsumamigui
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- obiyuta
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-10-
|
|
11
|
+
date: 2017-10-29 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: nokogiri
|
|
@@ -157,7 +157,9 @@ executables: []
|
|
|
157
157
|
extensions: []
|
|
158
158
|
extra_rdoc_files: []
|
|
159
159
|
files:
|
|
160
|
+
- ".circleci/config.yml"
|
|
160
161
|
- ".gitignore"
|
|
162
|
+
- ".hound.yml"
|
|
161
163
|
- ".pryrc"
|
|
162
164
|
- ".rspec"
|
|
163
165
|
- ".rubocop.yml"
|
|
@@ -169,7 +171,6 @@ files:
|
|
|
169
171
|
- Rakefile
|
|
170
172
|
- bin/console
|
|
171
173
|
- bin/setup
|
|
172
|
-
- circle.yml
|
|
173
174
|
- lib/tsumamigui.rb
|
|
174
175
|
- lib/tsumamigui/error.rb
|
|
175
176
|
- lib/tsumamigui/parser.rb
|
data/circle.yml
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
machine:
|
|
2
|
-
timezone: Asia/Tokyo
|
|
3
|
-
ruby:
|
|
4
|
-
version: 2.4.1
|
|
5
|
-
environment:
|
|
6
|
-
CC_TEST_REPORTER_ID: fb2b65ee179ec4ba3ac0f60d2047823566cc14dac0578c4c401532e682f5d435
|
|
7
|
-
database:
|
|
8
|
-
override:
|
|
9
|
-
- echo 'Skipping DB step.'
|
|
10
|
-
dependencies:
|
|
11
|
-
cache_directories:
|
|
12
|
-
- 'vendor/bundle'
|
|
13
|
-
- 'vendor/bundle_2.3.4'
|
|
14
|
-
- 'vendor/bundle_2.2.7'
|
|
15
|
-
- 'vendor/bundle_2.1.6'
|
|
16
|
-
pre:
|
|
17
|
-
- rvm install 2.3.4
|
|
18
|
-
- rvm install 2.2.7
|
|
19
|
-
- rvm install 2.1.6
|
|
20
|
-
override:
|
|
21
|
-
- bundle install --path vendor/bundle
|
|
22
|
-
- rvm-exec 2.3.4 bash -c 'bundle install --path vendor/bundle_2.3.4'
|
|
23
|
-
- rvm-exec 2.2.7 bash -c 'bundle install --path vendor/bundle_2.2.7'
|
|
24
|
-
- rvm-exec 2.1.6 bash -c 'bundle install --path vendor/bundle_2.1.6'
|
|
25
|
-
post:
|
|
26
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
|
27
|
-
- chmod +x ./cc-test-reporter
|
|
28
|
-
test:
|
|
29
|
-
pre:
|
|
30
|
-
- ./cc-test-reporter before-build
|
|
31
|
-
override:
|
|
32
|
-
- bundle config --local path vendor/bundle_2.3.4
|
|
33
|
-
- rvm-exec 2.3.4 bash -c 'bundle exec rspec'
|
|
34
|
-
- bundle config --local path vendor/bundle_2.2.7
|
|
35
|
-
- rvm-exec 2.2.7 bash -c 'bundle exec rspec'
|
|
36
|
-
- bundle config --local path vendor/bundle_2.1.6
|
|
37
|
-
- rvm-exec 2.1.6 bash -c 'bundle exec rspec'
|
|
38
|
-
- bundle config --local path vendor/bundle
|
|
39
|
-
- bundle exec rubocop
|
|
40
|
-
- bundle exec rspec
|
|
41
|
-
post:
|
|
42
|
-
- ./cc-test-reporter after-build --exit-code $EXIT_CODE
|
|
43
|
-
general:
|
|
44
|
-
artifacts:
|
|
45
|
-
- coverage
|