fetch_util 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +24 -10
- data/SKILL.md +10 -4
- data/lib/fetch_util/assets/extract.js +1 -1
- data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +5 -0
- data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +39 -0
- data/lib/fetch_util/browser/site_stabilization.rb +2 -0
- data/lib/fetch_util/browser/stabilization/page_flow.rb +39 -6
- data/lib/fetch_util/browser/stabilization/spa_hydration.rb +17 -0
- data/lib/fetch_util/cli.rb +37 -12
- data/lib/fetch_util/fetcher.rb +149 -2
- data/lib/fetch_util/result.rb +124 -9
- data/lib/fetch_util/search_transport.rb +496 -0
- data/lib/fetch_util/searcher.rb +51 -100
- data/lib/fetch_util/version.rb +1 -1
- data/lib/fetch_util.rb +1 -0
- metadata +3 -1
data/lib/fetch_util/result.rb
CHANGED
|
@@ -3,10 +3,13 @@
|
|
|
3
3
|
module FetchUtil
|
|
4
4
|
class Result
|
|
5
5
|
attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
|
|
6
|
-
:published_time, :canonical_url, :language, :
|
|
6
|
+
:published_time, :canonical_url, :language, :name, :company, :location,
|
|
7
|
+
:description, :ingredients, :instructions, :bedrooms, :bathrooms,
|
|
8
|
+
:area_sqft, :html, :markdown,
|
|
7
9
|
:metadata, :reader_mode, :content_type, :suspect, :warnings,
|
|
8
10
|
:content_completeness_ratio, :content_format, :paywall_state,
|
|
9
|
-
:
|
|
11
|
+
:price, :rating, :address, :social_kind, :platform, :handle,
|
|
12
|
+
:reply_count, :community, :score, :error_message
|
|
10
13
|
|
|
11
14
|
class << self
|
|
12
15
|
def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
|
|
@@ -22,7 +25,25 @@ module FetchUtil
|
|
|
22
25
|
warnings: warnings,
|
|
23
26
|
content_completeness_ratio: content_completeness_ratio,
|
|
24
27
|
content_format: content_format,
|
|
25
|
-
paywall_state: paywall_state
|
|
28
|
+
paywall_state: paywall_state,
|
|
29
|
+
name: payload["name"],
|
|
30
|
+
company: payload["company"],
|
|
31
|
+
location: payload["location"],
|
|
32
|
+
description: payload["description"],
|
|
33
|
+
ingredients: payload["ingredients"],
|
|
34
|
+
instructions: payload["instructions"],
|
|
35
|
+
bedrooms: payload["bedrooms"],
|
|
36
|
+
bathrooms: payload["bathrooms"],
|
|
37
|
+
area_sqft: payload["areaSqft"],
|
|
38
|
+
price: payload["price"],
|
|
39
|
+
rating: payload["rating"],
|
|
40
|
+
address: payload["address"],
|
|
41
|
+
social_kind: payload["socialKind"],
|
|
42
|
+
platform: payload["platform"],
|
|
43
|
+
handle: payload["handle"],
|
|
44
|
+
reply_count: payload["replyCount"],
|
|
45
|
+
community: payload["community"],
|
|
46
|
+
score: payload["score"]
|
|
26
47
|
)
|
|
27
48
|
|
|
28
49
|
new(
|
|
@@ -35,6 +56,15 @@ module FetchUtil
|
|
|
35
56
|
published_time: payload["publishedTime"],
|
|
36
57
|
canonical_url: canonical_url,
|
|
37
58
|
language: payload["language"],
|
|
59
|
+
name: payload["name"],
|
|
60
|
+
company: payload["company"],
|
|
61
|
+
location: payload["location"],
|
|
62
|
+
description: payload["description"],
|
|
63
|
+
ingredients: payload["ingredients"],
|
|
64
|
+
instructions: payload["instructions"],
|
|
65
|
+
bedrooms: payload["bedrooms"],
|
|
66
|
+
bathrooms: payload["bathrooms"],
|
|
67
|
+
area_sqft: payload["areaSqft"],
|
|
38
68
|
html: payload["html"],
|
|
39
69
|
markdown: payload["markdown"],
|
|
40
70
|
metadata: metadata,
|
|
@@ -44,7 +74,16 @@ module FetchUtil
|
|
|
44
74
|
warnings: warnings,
|
|
45
75
|
content_completeness_ratio: content_completeness_ratio,
|
|
46
76
|
content_format: content_format,
|
|
47
|
-
paywall_state: paywall_state
|
|
77
|
+
paywall_state: paywall_state,
|
|
78
|
+
price: payload["price"],
|
|
79
|
+
rating: payload["rating"],
|
|
80
|
+
address: payload["address"],
|
|
81
|
+
social_kind: payload["socialKind"],
|
|
82
|
+
platform: payload["platform"],
|
|
83
|
+
handle: payload["handle"],
|
|
84
|
+
reply_count: payload["replyCount"],
|
|
85
|
+
community: payload["community"],
|
|
86
|
+
score: payload["score"]
|
|
48
87
|
)
|
|
49
88
|
end
|
|
50
89
|
|
|
@@ -54,7 +93,13 @@ module FetchUtil
|
|
|
54
93
|
content_type: "error",
|
|
55
94
|
suspect: true,
|
|
56
95
|
warnings: [warning],
|
|
57
|
-
error_message: message
|
|
96
|
+
error_message: message,
|
|
97
|
+
social_kind: nil,
|
|
98
|
+
platform: nil,
|
|
99
|
+
handle: nil,
|
|
100
|
+
reply_count: nil,
|
|
101
|
+
community: nil,
|
|
102
|
+
score: nil
|
|
58
103
|
}.freeze
|
|
59
104
|
|
|
60
105
|
new(
|
|
@@ -67,6 +112,15 @@ module FetchUtil
|
|
|
67
112
|
published_time: nil,
|
|
68
113
|
canonical_url: nil,
|
|
69
114
|
language: nil,
|
|
115
|
+
name: nil,
|
|
116
|
+
company: nil,
|
|
117
|
+
location: nil,
|
|
118
|
+
description: nil,
|
|
119
|
+
ingredients: nil,
|
|
120
|
+
instructions: nil,
|
|
121
|
+
bedrooms: nil,
|
|
122
|
+
bathrooms: nil,
|
|
123
|
+
area_sqft: nil,
|
|
70
124
|
html: nil,
|
|
71
125
|
markdown: "",
|
|
72
126
|
metadata: metadata,
|
|
@@ -81,7 +135,10 @@ module FetchUtil
|
|
|
81
135
|
private
|
|
82
136
|
|
|
83
137
|
def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
|
|
84
|
-
content_completeness_ratio:, content_format:, paywall_state
|
|
138
|
+
content_completeness_ratio:, content_format:, paywall_state:,
|
|
139
|
+
name:, company:, location:, description:, ingredients:, instructions:,
|
|
140
|
+
bedrooms:, bathrooms:, area_sqft:, price:, rating:, address:,
|
|
141
|
+
social_kind:, platform:, handle:, reply_count:, community:, score:)
|
|
85
142
|
{
|
|
86
143
|
title: payload["title"],
|
|
87
144
|
byline: payload["byline"],
|
|
@@ -90,6 +147,15 @@ module FetchUtil
|
|
|
90
147
|
published_time: payload["publishedTime"],
|
|
91
148
|
canonical_url: canonical_url,
|
|
92
149
|
language: payload["language"],
|
|
150
|
+
name: name,
|
|
151
|
+
company: company,
|
|
152
|
+
location: location,
|
|
153
|
+
description: description,
|
|
154
|
+
ingredients: ingredients,
|
|
155
|
+
instructions: instructions,
|
|
156
|
+
bedrooms: bedrooms,
|
|
157
|
+
bathrooms: bathrooms,
|
|
158
|
+
area_sqft: area_sqft,
|
|
93
159
|
content_url: final_url,
|
|
94
160
|
reader_mode: payload["readerMode"],
|
|
95
161
|
content_type: content_type,
|
|
@@ -97,14 +163,27 @@ module FetchUtil
|
|
|
97
163
|
warnings: warnings,
|
|
98
164
|
content_completeness_ratio: content_completeness_ratio,
|
|
99
165
|
content_format: content_format,
|
|
100
|
-
paywall_state: paywall_state
|
|
166
|
+
paywall_state: paywall_state,
|
|
167
|
+
price: price,
|
|
168
|
+
rating: rating,
|
|
169
|
+
address: address,
|
|
170
|
+
social_kind: social_kind,
|
|
171
|
+
platform: platform,
|
|
172
|
+
handle: handle,
|
|
173
|
+
reply_count: reply_count,
|
|
174
|
+
community: community,
|
|
175
|
+
score: score
|
|
101
176
|
}.freeze
|
|
102
177
|
end
|
|
103
178
|
end
|
|
104
179
|
|
|
105
180
|
def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
|
|
106
|
-
canonical_url:, language:,
|
|
107
|
-
|
|
181
|
+
canonical_url:, language:, name:, company:, location:, description:, ingredients:,
|
|
182
|
+
instructions:, bedrooms:, bathrooms:, area_sqft:, html:, markdown:, metadata:,
|
|
183
|
+
reader_mode:, content_type:, suspect:, warnings:,
|
|
184
|
+
content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil,
|
|
185
|
+
price: nil, rating: nil, address: nil, social_kind: nil, platform: nil,
|
|
186
|
+
handle: nil, reply_count: nil, community: nil, score: nil, error_message: nil)
|
|
108
187
|
@url = url
|
|
109
188
|
@final_url = final_url
|
|
110
189
|
@title = title
|
|
@@ -114,6 +193,15 @@ module FetchUtil
|
|
|
114
193
|
@published_time = published_time
|
|
115
194
|
@canonical_url = canonical_url
|
|
116
195
|
@language = language
|
|
196
|
+
@name = name
|
|
197
|
+
@company = company
|
|
198
|
+
@location = location
|
|
199
|
+
@description = description
|
|
200
|
+
@ingredients = ingredients
|
|
201
|
+
@instructions = instructions
|
|
202
|
+
@bedrooms = bedrooms
|
|
203
|
+
@bathrooms = bathrooms
|
|
204
|
+
@area_sqft = area_sqft
|
|
117
205
|
@html = html
|
|
118
206
|
@markdown = markdown
|
|
119
207
|
@metadata = metadata.freeze
|
|
@@ -124,6 +212,15 @@ module FetchUtil
|
|
|
124
212
|
@content_completeness_ratio = content_completeness_ratio
|
|
125
213
|
@content_format = content_format&.freeze
|
|
126
214
|
@paywall_state = paywall_state&.freeze
|
|
215
|
+
@price = price
|
|
216
|
+
@rating = rating
|
|
217
|
+
@address = address
|
|
218
|
+
@social_kind = social_kind
|
|
219
|
+
@platform = platform
|
|
220
|
+
@handle = handle
|
|
221
|
+
@reply_count = reply_count
|
|
222
|
+
@community = community
|
|
223
|
+
@score = score
|
|
127
224
|
@error_message = error_message
|
|
128
225
|
end
|
|
129
226
|
|
|
@@ -138,6 +235,15 @@ module FetchUtil
|
|
|
138
235
|
published_time: published_time,
|
|
139
236
|
canonical_url: canonical_url,
|
|
140
237
|
language: language,
|
|
238
|
+
name: name,
|
|
239
|
+
company: company,
|
|
240
|
+
location: location,
|
|
241
|
+
description: description,
|
|
242
|
+
ingredients: ingredients,
|
|
243
|
+
instructions: instructions,
|
|
244
|
+
bedrooms: bedrooms,
|
|
245
|
+
bathrooms: bathrooms,
|
|
246
|
+
area_sqft: area_sqft,
|
|
141
247
|
html: html,
|
|
142
248
|
markdown: markdown,
|
|
143
249
|
metadata: metadata,
|
|
@@ -148,6 +254,15 @@ module FetchUtil
|
|
|
148
254
|
content_completeness_ratio: content_completeness_ratio,
|
|
149
255
|
content_format: content_format,
|
|
150
256
|
paywall_state: paywall_state,
|
|
257
|
+
price: price,
|
|
258
|
+
rating: rating,
|
|
259
|
+
address: address,
|
|
260
|
+
social_kind: social_kind,
|
|
261
|
+
platform: platform,
|
|
262
|
+
handle: handle,
|
|
263
|
+
reply_count: reply_count,
|
|
264
|
+
community: community,
|
|
265
|
+
score: score,
|
|
151
266
|
error_message: error_message
|
|
152
267
|
}
|
|
153
268
|
end
|