spidy 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +53 -8
- data/Gemfile +18 -4
- data/Gemfile.lock +71 -19
- data/README.md +95 -16
- data/Rakefile +0 -2
- data/example/check_ferrum.rb +114 -0
- data/example/check_lightpanda.rb +59 -0
- data/example/connect_test.rb +48 -0
- data/example/lightpanda_links.rb +80 -0
- data/example/master_detail.rb +1 -3
- data/example/proxy.rb +0 -2
- data/example/retry.rb +0 -2
- data/example/run_with_lightpanda.rb +25 -0
- data/example/simple_test.rb +53 -0
- data/example/test_lightpanda.rb +86 -0
- data/example/wikip.rb +2 -4
- data/exe/spidy +0 -2
- data/lib/spidy/binder/error.rb +0 -2
- data/lib/spidy/binder/html.rb +0 -2
- data/lib/spidy/binder/json.rb +0 -2
- data/lib/spidy/binder/xml.rb +0 -2
- data/lib/spidy/binder.rb +0 -2
- data/lib/spidy/command_line.rb +4 -6
- data/lib/spidy/connector/direct.rb +0 -2
- data/lib/spidy/connector/html.rb +0 -2
- data/lib/spidy/connector/json.rb +2 -4
- data/lib/spidy/connector/lightpanda.rb +161 -0
- data/lib/spidy/connector/xml.rb +4 -6
- data/lib/spidy/connector.rb +7 -8
- data/lib/spidy/console.rb +0 -2
- data/lib/spidy/definition.rb +2 -4
- data/lib/spidy/definition_file.rb +0 -2
- data/lib/spidy/definition_object.rb +0 -2
- data/lib/spidy/shell.rb +0 -2
- data/lib/spidy/spider.rb +2 -4
- data/lib/spidy/version.rb +1 -3
- data/lib/spidy.rb +3 -5
- data/spidy.gemspec +4 -15
- metadata +11 -102
- data/.rubocop_todo.yml +0 -13
data/lib/spidy/version.rb
CHANGED
data/lib/spidy.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'spidy/version'
|
1
|
+
require_relative 'spidy/version'
|
4
2
|
require 'active_support/all'
|
5
3
|
require 'mechanize'
|
6
4
|
require 'open-uri'
|
@@ -29,11 +27,11 @@ module Spidy
|
|
29
27
|
Spidy::DefinitionFile.open(filepath).spidy
|
30
28
|
end
|
31
29
|
|
32
|
-
def self.define(&
|
30
|
+
def self.define(&)
|
33
31
|
spidy = Module.new do
|
34
32
|
class_eval do
|
35
33
|
extend ::Spidy::Definition
|
36
|
-
module_eval(&
|
34
|
+
module_eval(&)
|
37
35
|
end
|
38
36
|
end
|
39
37
|
spidy.instance_eval do
|
data/spidy.gemspec
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
lib = File.expand_path('lib', __dir__)
|
4
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
3
|
require 'spidy/version'
|
@@ -23,19 +21,10 @@ Gem::Specification.new do |spec|
|
|
23
21
|
spec.bindir = 'exe'
|
24
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
23
|
spec.require_paths = ['lib']
|
26
|
-
|
27
|
-
spec.
|
28
|
-
spec.
|
29
|
-
spec.
|
30
|
-
spec.add_development_dependency 'rake', '~> 13.0'
|
31
|
-
spec.add_development_dependency 'rspec', '~> 3.0'
|
32
|
-
spec.add_development_dependency 'rspec-command'
|
33
|
-
spec.add_development_dependency 'sinatra'
|
34
|
-
|
35
|
-
spec.add_runtime_dependency 'activesupport', '~> 7.1'
|
36
|
-
spec.add_runtime_dependency 'mechanize'
|
37
|
-
spec.add_runtime_dependency 'socksify'
|
38
|
-
spec.add_runtime_dependency 'tor'
|
24
|
+
spec.add_dependency 'activesupport', '~> 7.1'
|
25
|
+
spec.add_dependency 'mechanize'
|
26
|
+
spec.add_dependency 'socksify'
|
27
|
+
spec.add_dependency 'tor'
|
39
28
|
spec.metadata = {
|
40
29
|
'rubygems_mfa_required' => 'true'
|
41
30
|
}
|
metadata
CHANGED
@@ -1,112 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spidy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- aileron
|
8
8
|
bindir: exe
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-
|
10
|
+
date: 2025-04-15 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
|
-
- !ruby/object:Gem::Dependency
|
13
|
-
name: bundler
|
14
|
-
requirement: !ruby/object:Gem::Requirement
|
15
|
-
requirements:
|
16
|
-
- - "~>"
|
17
|
-
- !ruby/object:Gem::Version
|
18
|
-
version: '2.0'
|
19
|
-
type: :development
|
20
|
-
prerelease: false
|
21
|
-
version_requirements: !ruby/object:Gem::Requirement
|
22
|
-
requirements:
|
23
|
-
- - "~>"
|
24
|
-
- !ruby/object:Gem::Version
|
25
|
-
version: '2.0'
|
26
|
-
- !ruby/object:Gem::Dependency
|
27
|
-
name: capybara_discoball
|
28
|
-
requirement: !ruby/object:Gem::Requirement
|
29
|
-
requirements:
|
30
|
-
- - ">="
|
31
|
-
- !ruby/object:Gem::Version
|
32
|
-
version: '0'
|
33
|
-
type: :development
|
34
|
-
prerelease: false
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
36
|
-
requirements:
|
37
|
-
- - ">="
|
38
|
-
- !ruby/object:Gem::Version
|
39
|
-
version: '0'
|
40
|
-
- !ruby/object:Gem::Dependency
|
41
|
-
name: ffaker
|
42
|
-
requirement: !ruby/object:Gem::Requirement
|
43
|
-
requirements:
|
44
|
-
- - ">="
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
version: '0'
|
47
|
-
type: :development
|
48
|
-
prerelease: false
|
49
|
-
version_requirements: !ruby/object:Gem::Requirement
|
50
|
-
requirements:
|
51
|
-
- - ">="
|
52
|
-
- !ruby/object:Gem::Version
|
53
|
-
version: '0'
|
54
|
-
- !ruby/object:Gem::Dependency
|
55
|
-
name: rake
|
56
|
-
requirement: !ruby/object:Gem::Requirement
|
57
|
-
requirements:
|
58
|
-
- - "~>"
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '13.0'
|
61
|
-
type: :development
|
62
|
-
prerelease: false
|
63
|
-
version_requirements: !ruby/object:Gem::Requirement
|
64
|
-
requirements:
|
65
|
-
- - "~>"
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
version: '13.0'
|
68
|
-
- !ruby/object:Gem::Dependency
|
69
|
-
name: rspec
|
70
|
-
requirement: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - "~>"
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '3.0'
|
75
|
-
type: :development
|
76
|
-
prerelease: false
|
77
|
-
version_requirements: !ruby/object:Gem::Requirement
|
78
|
-
requirements:
|
79
|
-
- - "~>"
|
80
|
-
- !ruby/object:Gem::Version
|
81
|
-
version: '3.0'
|
82
|
-
- !ruby/object:Gem::Dependency
|
83
|
-
name: rspec-command
|
84
|
-
requirement: !ruby/object:Gem::Requirement
|
85
|
-
requirements:
|
86
|
-
- - ">="
|
87
|
-
- !ruby/object:Gem::Version
|
88
|
-
version: '0'
|
89
|
-
type: :development
|
90
|
-
prerelease: false
|
91
|
-
version_requirements: !ruby/object:Gem::Requirement
|
92
|
-
requirements:
|
93
|
-
- - ">="
|
94
|
-
- !ruby/object:Gem::Version
|
95
|
-
version: '0'
|
96
|
-
- !ruby/object:Gem::Dependency
|
97
|
-
name: sinatra
|
98
|
-
requirement: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - ">="
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: '0'
|
103
|
-
type: :development
|
104
|
-
prerelease: false
|
105
|
-
version_requirements: !ruby/object:Gem::Requirement
|
106
|
-
requirements:
|
107
|
-
- - ">="
|
108
|
-
- !ruby/object:Gem::Version
|
109
|
-
version: '0'
|
110
12
|
- !ruby/object:Gem::Dependency
|
111
13
|
name: activesupport
|
112
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -173,7 +75,6 @@ files:
|
|
173
75
|
- ".gitignore"
|
174
76
|
- ".rspec"
|
175
77
|
- ".rubocop.yml"
|
176
|
-
- ".rubocop_todo.yml"
|
177
78
|
- ".ruby-version"
|
178
79
|
- ".travis.yml"
|
179
80
|
- CHANGELOG.md
|
@@ -186,9 +87,16 @@ files:
|
|
186
87
|
- Rakefile
|
187
88
|
- bin/console
|
188
89
|
- bin/setup
|
90
|
+
- example/check_ferrum.rb
|
91
|
+
- example/check_lightpanda.rb
|
92
|
+
- example/connect_test.rb
|
93
|
+
- example/lightpanda_links.rb
|
189
94
|
- example/master_detail.rb
|
190
95
|
- example/proxy.rb
|
191
96
|
- example/retry.rb
|
97
|
+
- example/run_with_lightpanda.rb
|
98
|
+
- example/simple_test.rb
|
99
|
+
- example/test_lightpanda.rb
|
192
100
|
- example/wikip.rb
|
193
101
|
- exe/spidy
|
194
102
|
- lib/spidy.rb
|
@@ -202,6 +110,7 @@ files:
|
|
202
110
|
- lib/spidy/connector/direct.rb
|
203
111
|
- lib/spidy/connector/html.rb
|
204
112
|
- lib/spidy/connector/json.rb
|
113
|
+
- lib/spidy/connector/lightpanda.rb
|
205
114
|
- lib/spidy/connector/xml.rb
|
206
115
|
- lib/spidy/console.rb
|
207
116
|
- lib/spidy/definition.rb
|
@@ -231,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
231
140
|
- !ruby/object:Gem::Version
|
232
141
|
version: '0'
|
233
142
|
requirements: []
|
234
|
-
rubygems_version: 3.6.
|
143
|
+
rubygems_version: 3.6.6
|
235
144
|
specification_version: 4
|
236
145
|
summary: web spider dsl
|
237
146
|
test_files: []
|
data/.rubocop_todo.yml
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
# This configuration was generated by
|
2
|
-
# `rubocop --auto-gen-config`
|
3
|
-
# on 2019-03-29 18:00:03 +0900 using RuboCop version 0.66.0.
|
4
|
-
# The point is for the user to remove these configuration records
|
5
|
-
# one by one as the offenses are removed from the code base.
|
6
|
-
# Note that changes in the inspected code, or installation of new
|
7
|
-
# versions of RuboCop, may require this file to be generated again.
|
8
|
-
|
9
|
-
# Offense count: 7
|
10
|
-
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
11
|
-
# URISchemes: http, https
|
12
|
-
Metrics/LineLength:
|
13
|
-
Max: 96
|