wgit 0.10.8 → 0.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (20) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +33 -1
  3. data/CONTRIBUTING.md +1 -1
  4. data/README.md +3 -1
  5. data/bin/wgit +3 -1
  6. data/lib/wgit/assertable.rb +2 -2
  7. data/lib/wgit/crawler.rb +56 -34
  8. data/lib/wgit/database/database.rb +64 -52
  9. data/lib/wgit/document.rb +54 -40
  10. data/lib/wgit/document_extractors.rb +15 -1
  11. data/lib/wgit/dsl.rb +16 -20
  12. data/lib/wgit/indexer.rb +157 -63
  13. data/lib/wgit/logger.rb +1 -1
  14. data/lib/wgit/response.rb +21 -6
  15. data/lib/wgit/robots_parser.rb +193 -0
  16. data/lib/wgit/url.rb +115 -55
  17. data/lib/wgit/utils.rb +81 -28
  18. data/lib/wgit/version.rb +1 -1
  19. data/lib/wgit.rb +1 -0
  20. metadata +33 -38
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wgit
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.8
4
+ version: 0.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Telford
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-08-18 00:00:00.000000000 Z
11
+ date: 2024-01-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -16,188 +16,182 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.6'
19
+ version: '2.8'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.6'
26
+ version: '2.8'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: mongo
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '2.9'
33
+ version: '2.19'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '2.9'
40
+ version: '2.19'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: nokogiri
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '1.10'
47
+ version: '1.15'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '1.10'
54
+ version: '1.15'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: typhoeus
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '1.3'
61
+ version: '1.4'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '1.3'
68
+ version: '1.4'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: ferrum
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '0.8'
75
+ version: '0.14'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '0.8'
82
+ version: '0.14'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: byebug
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '10.0'
89
+ version: '11.1'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '10.0'
96
+ version: '11.1'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: dotenv
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '2.5'
103
+ version: '2.8'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '2.5'
110
+ version: '2.8'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: maxitest
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '3.3'
117
+ version: '5.4'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '3.3'
124
+ version: '5.4'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: pry
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '0.12'
131
+ version: '0.14'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '0.12'
138
+ version: '0.14'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: rubocop
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '0.74'
145
+ version: '1.57'
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '0.74'
152
+ version: '1.57'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: toys
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '0.8'
159
+ version: '0.15'
160
160
  type: :development
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '0.8'
166
+ version: '0.15'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: webmock
169
169
  requirement: !ruby/object:Gem::Requirement
170
170
  requirements:
171
171
  - - "~>"
172
172
  - !ruby/object:Gem::Version
173
- version: '3.6'
173
+ version: '3.19'
174
174
  type: :development
175
175
  prerelease: false
176
176
  version_requirements: !ruby/object:Gem::Requirement
177
177
  requirements:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
- version: '3.6'
180
+ version: '3.19'
181
181
  - !ruby/object:Gem::Dependency
182
182
  name: yard
183
183
  requirement: !ruby/object:Gem::Requirement
184
184
  requirements:
185
- - - ">="
186
- - !ruby/object:Gem::Version
187
- version: 0.9.20
188
- - - "<"
185
+ - - "~>"
189
186
  - !ruby/object:Gem::Version
190
- version: '1.0'
187
+ version: '0.9'
191
188
  type: :development
192
189
  prerelease: false
193
190
  version_requirements: !ruby/object:Gem::Requirement
194
191
  requirements:
195
- - - ">="
196
- - !ruby/object:Gem::Version
197
- version: 0.9.20
198
- - - "<"
192
+ - - "~>"
199
193
  - !ruby/object:Gem::Version
200
- version: '1.0'
194
+ version: '0.9'
201
195
  description: 'Wgit was primarily designed to crawl static HTML websites to index and
202
196
  search their content - providing the basis of any search engine; but Wgit is suitable
203
197
  for many application domains including: URL parsing, data mining and statistical
@@ -223,6 +217,7 @@ files:
223
217
  - "./lib/wgit/indexer.rb"
224
218
  - "./lib/wgit/logger.rb"
225
219
  - "./lib/wgit/response.rb"
220
+ - "./lib/wgit/robots_parser.rb"
226
221
  - "./lib/wgit/url.rb"
227
222
  - "./lib/wgit/utils.rb"
228
223
  - "./lib/wgit/version.rb"
@@ -251,7 +246,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
251
246
  requirements:
252
247
  - - ">="
253
248
  - !ruby/object:Gem::Version
254
- version: '2.6'
249
+ version: '3'
255
250
  - - "<"
256
251
  - !ruby/object:Gem::Version
257
252
  version: '4'
@@ -261,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
261
256
  - !ruby/object:Gem::Version
262
257
  version: '0'
263
258
  requirements: []
264
- rubygems_version: 3.2.22
259
+ rubygems_version: 3.5.3
265
260
  signing_key:
266
261
  specification_version: 4
267
262
  summary: Wgit is a HTML web crawler, written in Ruby, that allows you to programmatically