spidy 0.3.3 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/example/wikip.rb +22 -0
- data/lib/spidy/connector.rb +2 -0
- data/lib/spidy/connector/direct.rb +5 -1
- data/lib/spidy/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 38936d0a025edde8a0ecb15a164c36fbccf1346f9bee9ec7d01a43931eaeb163
|
|
4
|
+
data.tar.gz: 71c02d424da0b18cfad383035c78465b1c81ccdfe41399baa511a7bae3675662
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d97c61f76fb9dde1d5693c7d33830c6df7605930408c42666ffe31a6d46262d1203cfed710d2112f2c396cdcbc3e40f1149d9fc005c5947af3851946bd6f487e
|
|
7
|
+
data.tar.gz: ffe6d6362f6bb50c11e54cb398730b0a649ac832b5fdd048d61675dce7a17ec58993900cbaa250238c0cc702cbe649e901cab660c265adb2ef8207e2704ea48c
|
data/Gemfile.lock
CHANGED
data/example/wikip.rb
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Spidy.define do
|
|
4
|
+
def self.infobox_scrape(params, &block)
|
|
5
|
+
call(params.html.at('.infobox'), name: :infobox, &block)
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
define(as: :html) do
|
|
9
|
+
let(:title, 'h1')
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
define(:infobox, as: :html, connector: :direct) do
|
|
13
|
+
let(:columns) do
|
|
14
|
+
html.search('tr').each do |tr|
|
|
15
|
+
{
|
|
16
|
+
name: tr.at('th')&.text,
|
|
17
|
+
value: tr.at('td')&.text
|
|
18
|
+
}
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
data/lib/spidy/connector.rb
CHANGED
|
@@ -126,6 +126,8 @@ module Spidy::Connector
|
|
|
126
126
|
wait_time ||= DEFAULT_WAIT_TIME
|
|
127
127
|
|
|
128
128
|
connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
|
|
129
|
+
return connector if connector.is_a?(Spidy::Connector::Direct)
|
|
130
|
+
|
|
129
131
|
RetryableCaller.new(connector, wait_time: wait_time, logger: logger)
|
|
130
132
|
end
|
|
131
133
|
|
data/lib/spidy/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: spidy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- aileron
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-10-
|
|
11
|
+
date: 2020-10-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -218,6 +218,7 @@ files:
|
|
|
218
218
|
- example/master_detail.rb
|
|
219
219
|
- example/proxy.rb
|
|
220
220
|
- example/retry.rb
|
|
221
|
+
- example/wikip.rb
|
|
221
222
|
- exe/spidy
|
|
222
223
|
- lib/spidy.rb
|
|
223
224
|
- lib/spidy/binder.rb
|
|
@@ -257,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
257
258
|
- !ruby/object:Gem::Version
|
|
258
259
|
version: '0'
|
|
259
260
|
requirements: []
|
|
260
|
-
rubygems_version: 3.
|
|
261
|
+
rubygems_version: 3.1.4
|
|
261
262
|
signing_key:
|
|
262
263
|
specification_version: 4
|
|
263
264
|
summary: web spider dsl
|