tsumamigui 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +109 -0
- data/.hound.yml +2 -0
- data/.rubocop.yml +3 -1
- data/CHANGELOG.md +10 -0
- data/README.md +1 -1
- data/lib/tsumamigui/parser.rb +4 -8
- data/lib/tsumamigui/request.rb +5 -10
- data/lib/tsumamigui/version.rb +1 -1
- metadata +4 -3
- data/circle.yml +0 -45
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: beb1d6f4f9c9c2df335b58c81ae398a4428845fa
|
|
4
|
+
data.tar.gz: 2ae6515d29e64f28cb8e7f58c652d2cb61b1c278
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc852aee3ae850519d9de601a158c5158f3f7574d34cee4ef39e563a0604a403463d5d34b072350084ddb93ef156fe7df8d9024cec730fc6f2c9f56680515ac9
|
|
7
|
+
data.tar.gz: 236839d9eeadd3c4da5f416a34debf19361e042f50abdc2da525ea42c56c787312aa586ec24837aba153be366c8aad0fcffed216f6f2845b1482dfb2feb1041a
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
|
|
2
|
+
defaults: &defaults
|
|
3
|
+
working_directory: ~/tsumamigui
|
|
4
|
+
environment:
|
|
5
|
+
CC_TEST_REPORTER_ID: fb2b65ee179ec4ba3ac0f60d2047823566cc14dac0578c4c401532e682f5d435
|
|
6
|
+
|
|
7
|
+
jobs: &jobs
|
|
8
|
+
jobs:
|
|
9
|
+
- build
|
|
10
|
+
- lint:
|
|
11
|
+
requires:
|
|
12
|
+
- build
|
|
13
|
+
- rspec:
|
|
14
|
+
requires:
|
|
15
|
+
- build
|
|
16
|
+
- rspec-2_3_5
|
|
17
|
+
- rspec-2_2_8
|
|
18
|
+
|
|
19
|
+
version: 2
|
|
20
|
+
jobs:
|
|
21
|
+
build:
|
|
22
|
+
<<: *defaults
|
|
23
|
+
docker:
|
|
24
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
25
|
+
steps:
|
|
26
|
+
- checkout
|
|
27
|
+
- restore_cache:
|
|
28
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
29
|
+
- run: bundle install --path vendor/bundle
|
|
30
|
+
- save_cache:
|
|
31
|
+
paths:
|
|
32
|
+
- vendor/bundle
|
|
33
|
+
- Gemfile.lock
|
|
34
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
35
|
+
|
|
36
|
+
lint:
|
|
37
|
+
docker:
|
|
38
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
39
|
+
steps:
|
|
40
|
+
- checkout
|
|
41
|
+
- restore_cache:
|
|
42
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
43
|
+
- run: bundle install --path vendor/bundle
|
|
44
|
+
- run: bundle exec rubocop
|
|
45
|
+
|
|
46
|
+
rspec:
|
|
47
|
+
<<: *defaults
|
|
48
|
+
docker:
|
|
49
|
+
- image: circleci/ruby:2.4.2-node-browsers
|
|
50
|
+
steps:
|
|
51
|
+
- checkout
|
|
52
|
+
- restore_cache:
|
|
53
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-latest
|
|
54
|
+
|
|
55
|
+
- run: curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
|
56
|
+
- run: chmod +x ./cc-test-reporter
|
|
57
|
+
- run: bundle install --path vendor/bundle
|
|
58
|
+
- run: bundle exec rspec
|
|
59
|
+
- run: ./cc-test-reporter after-build -t simplecov --exit-code $?
|
|
60
|
+
- store_artifacts:
|
|
61
|
+
path: coverage
|
|
62
|
+
|
|
63
|
+
rspec-2_3_5:
|
|
64
|
+
<<: *defaults
|
|
65
|
+
docker:
|
|
66
|
+
- image: circleci/ruby:2.3.5-node-browsers
|
|
67
|
+
steps:
|
|
68
|
+
- checkout
|
|
69
|
+
- restore_cache:
|
|
70
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_3_5
|
|
71
|
+
- run: bundle install --path vendor/bundle
|
|
72
|
+
- run: bundle exec rspec
|
|
73
|
+
- save_cache:
|
|
74
|
+
paths:
|
|
75
|
+
- vendor/bundle
|
|
76
|
+
- Gemfile.lock
|
|
77
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_3_5
|
|
78
|
+
|
|
79
|
+
rspec-2_2_8:
|
|
80
|
+
<<: *defaults
|
|
81
|
+
docker:
|
|
82
|
+
- image: circleci/ruby:2.2.8-node-browsers
|
|
83
|
+
steps:
|
|
84
|
+
- checkout
|
|
85
|
+
- restore_cache:
|
|
86
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_2_8
|
|
87
|
+
- run: bundle install --path vendor/bundle
|
|
88
|
+
- run: bundle exec rspec
|
|
89
|
+
- save_cache:
|
|
90
|
+
paths:
|
|
91
|
+
- vendor/bundle
|
|
92
|
+
- Gemfile.lock
|
|
93
|
+
key: gems-{{ checksum "Gemfile" }}-{{ .Revision }}-2_2_8
|
|
94
|
+
|
|
95
|
+
workflows:
|
|
96
|
+
version: 2
|
|
97
|
+
build_and_test:
|
|
98
|
+
<<: *jobs
|
|
99
|
+
|
|
100
|
+
scheduled-build_and_test:
|
|
101
|
+
triggers:
|
|
102
|
+
- schedule:
|
|
103
|
+
cron: "0 23 * * *"
|
|
104
|
+
filters:
|
|
105
|
+
branches:
|
|
106
|
+
only:
|
|
107
|
+
- master
|
|
108
|
+
<<: *jobs
|
|
109
|
+
|
data/.hound.yml
ADDED
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Tsumamigui
|
|
2
|
-
[](https://badge.fury.io/rb/tsumamigui) [](https://circleci.com/gh/obiyuta/tsumamigui/tree/master) [](https://codeclimate.com/github/obiyuta/tsumamigui) [](https://codeclimate.com/github/obiyuta/tsumamigui/coverage) [](https://gemnasium.com/github.com/obiyuta/tsumamigui) [](http://inch-ci.org/github/obiyuta/tsumamigui)
|
|
2
|
+
[](https://badge.fury.io/rb/tsumamigui) [](https://circleci.com/gh/obiyuta/tsumamigui/tree/master) [](https://codeclimate.com/github/obiyuta/tsumamigui) [](https://codeclimate.com/github/obiyuta/tsumamigui/coverage) [](https://gemnasium.com/github.com/obiyuta/tsumamigui) [](http://inch-ci.org/github/obiyuta/tsumamigui) [](https://codebeat.co/projects/github-com-obiyuta-tsumamigui-master)
|
|
3
3
|
|
|
4
4
|
Tsumamigui（つまみぐい） is a simple and hussle-free Ruby web scraping library.
|
|
5
5
|
|
data/lib/tsumamigui/parser.rb
CHANGED
|
@@ -27,14 +27,12 @@ module Tsumamigui
|
|
|
27
27
|
# @return [Array<Hash>] parsed responses
|
|
28
28
|
# @raise [Tsumamigui::ParserError]
|
|
29
29
|
def parse(responses)
|
|
30
|
-
|
|
31
|
-
responses.each do |res|
|
|
30
|
+
responses.map do |res|
|
|
32
31
|
url, html, charset = res.to_array
|
|
33
32
|
result = extract(Nokogiri::HTML.parse(html, nil, charset))
|
|
34
33
|
result[:scraped_from] = url
|
|
35
|
-
|
|
34
|
+
result
|
|
36
35
|
end
|
|
37
|
-
results
|
|
38
36
|
rescue => e
|
|
39
37
|
raise ParserError, e.message
|
|
40
38
|
end
|
|
@@ -44,11 +42,9 @@ module Tsumamigui
|
|
|
44
42
|
# @return [Hash] xpath and its key
|
|
45
43
|
# @raise [Tsumamigui::ParserError]
|
|
46
44
|
def extract(document)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
result[k] = document.xpath(v).to_s
|
|
45
|
+
@xpath.each_with_object({}) do |(key, value), hash|
|
|
46
|
+
hash[key] = document.xpath(value).to_s
|
|
50
47
|
end
|
|
51
|
-
result
|
|
52
48
|
rescue => e
|
|
53
49
|
raise ParserError, e.message
|
|
54
50
|
end
|
data/lib/tsumamigui/request.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Tsumamigui
|
|
|
9
9
|
# @param [Array<String>] urls
|
|
10
10
|
# @return [Array<Tsumamigui::Response>] response instances
|
|
11
11
|
def run(*urls)
|
|
12
|
-
new(urls).
|
|
12
|
+
new(urls).exec_request
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -22,21 +22,16 @@ module Tsumamigui
|
|
|
22
22
|
def initialize(*urls)
|
|
23
23
|
@urls = urls.flatten
|
|
24
24
|
raise ArgumentError, 'No argument is specified' if @urls.empty?
|
|
25
|
-
@urls
|
|
26
25
|
end
|
|
27
26
|
|
|
28
|
-
private
|
|
29
|
-
|
|
30
27
|
# Execute request and return responses
|
|
31
28
|
# @return [Array<Tsumamigui::Response>] response instances
|
|
32
|
-
def
|
|
33
|
-
|
|
34
|
-
@urls.each do |url|
|
|
35
|
-
results.push(fetch(url))
|
|
36
|
-
end
|
|
37
|
-
results
|
|
29
|
+
def exec_request
|
|
30
|
+
@urls.map { |url| fetch(url) }
|
|
38
31
|
end
|
|
39
32
|
|
|
33
|
+
private
|
|
34
|
+
|
|
40
35
|
# Fetch HTML from url and convert to response object
|
|
41
36
|
# @param [String] url
|
|
42
37
|
# @return [Tsumamigui::Response] response
|
data/lib/tsumamigui/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tsumamigui
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- obiyuta
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-10-
|
|
11
|
+
date: 2017-10-29 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: nokogiri
|
|
@@ -157,7 +157,9 @@ executables: []
|
|
|
157
157
|
extensions: []
|
|
158
158
|
extra_rdoc_files: []
|
|
159
159
|
files:
|
|
160
|
+
- ".circleci/config.yml"
|
|
160
161
|
- ".gitignore"
|
|
162
|
+
- ".hound.yml"
|
|
161
163
|
- ".pryrc"
|
|
162
164
|
- ".rspec"
|
|
163
165
|
- ".rubocop.yml"
|
|
@@ -169,7 +171,6 @@ files:
|
|
|
169
171
|
- Rakefile
|
|
170
172
|
- bin/console
|
|
171
173
|
- bin/setup
|
|
172
|
-
- circle.yml
|
|
173
174
|
- lib/tsumamigui.rb
|
|
174
175
|
- lib/tsumamigui/error.rb
|
|
175
176
|
- lib/tsumamigui/parser.rb
|
data/circle.yml
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
machine:
|
|
2
|
-
timezone: Asia/Tokyo
|
|
3
|
-
ruby:
|
|
4
|
-
version: 2.4.1
|
|
5
|
-
environment:
|
|
6
|
-
CC_TEST_REPORTER_ID: fb2b65ee179ec4ba3ac0f60d2047823566cc14dac0578c4c401532e682f5d435
|
|
7
|
-
database:
|
|
8
|
-
override:
|
|
9
|
-
- echo 'Skipping DB step.'
|
|
10
|
-
dependencies:
|
|
11
|
-
cache_directories:
|
|
12
|
-
- 'vendor/bundle'
|
|
13
|
-
- 'vendor/bundle_2.3.4'
|
|
14
|
-
- 'vendor/bundle_2.2.7'
|
|
15
|
-
- 'vendor/bundle_2.1.6'
|
|
16
|
-
pre:
|
|
17
|
-
- rvm install 2.3.4
|
|
18
|
-
- rvm install 2.2.7
|
|
19
|
-
- rvm install 2.1.6
|
|
20
|
-
override:
|
|
21
|
-
- bundle install --path vendor/bundle
|
|
22
|
-
- rvm-exec 2.3.4 bash -c 'bundle install --path vendor/bundle_2.3.4'
|
|
23
|
-
- rvm-exec 2.2.7 bash -c 'bundle install --path vendor/bundle_2.2.7'
|
|
24
|
-
- rvm-exec 2.1.6 bash -c 'bundle install --path vendor/bundle_2.1.6'
|
|
25
|
-
post:
|
|
26
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
|
27
|
-
- chmod +x ./cc-test-reporter
|
|
28
|
-
test:
|
|
29
|
-
pre:
|
|
30
|
-
- ./cc-test-reporter before-build
|
|
31
|
-
override:
|
|
32
|
-
- bundle config --local path vendor/bundle_2.3.4
|
|
33
|
-
- rvm-exec 2.3.4 bash -c 'bundle exec rspec'
|
|
34
|
-
- bundle config --local path vendor/bundle_2.2.7
|
|
35
|
-
- rvm-exec 2.2.7 bash -c 'bundle exec rspec'
|
|
36
|
-
- bundle config --local path vendor/bundle_2.1.6
|
|
37
|
-
- rvm-exec 2.1.6 bash -c 'bundle exec rspec'
|
|
38
|
-
- bundle config --local path vendor/bundle
|
|
39
|
-
- bundle exec rubocop
|
|
40
|
-
- bundle exec rspec
|
|
41
|
-
post:
|
|
42
|
-
- ./cc-test-reporter after-build --exit-code $EXIT_CODE
|
|
43
|
-
general:
|
|
44
|
-
artifacts:
|
|
45
|
-
- coverage
|