pulse-downloader 0.1.15 → 0.1.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pulse/downloader/version.rb +1 -1
- data/lib/pulse/downloader/web_page_parser.rb +14 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2b4edaf42a90782470ab4956cc6d9cf4214d6bb81c03b8e5b55193ac1867330e
|
|
4
|
+
data.tar.gz: 81da683b9c0aa82f17197a280318b529ab4f6630dd20fa385925739f02165ca2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d8771a9ece20d74d44bb8e365d7f51be288ee8a9f5ff5dbec826013aec57998147d58d0e17ffd6ab5c7213f2dc441bfd4b471716f99e8df40e06cf2c7c00ddf0
|
|
7
|
+
data.tar.gz: 053c925e99adeeb3cae3f277bb0e3ceda00d327ac57fdc76ce25e9a86220d66a291c0f388aa1aebe66a92e7b9b8971409c7b59dc4d188fa87ee4504914b52052
|
|
@@ -29,21 +29,34 @@ module Pulse
|
|
|
29
29
|
.map { |link| link['href'] }
|
|
30
30
|
.compact
|
|
31
31
|
.select { |link| link.include? file_type }
|
|
32
|
+
.map { |link| add_base_url(link) }
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
def extract_embedded_images(response)
|
|
36
|
+
return [] unless scrape_images
|
|
37
|
+
|
|
35
38
|
parse_html(response.body)
|
|
36
39
|
.css('img')
|
|
37
40
|
.to_a
|
|
38
41
|
.map { |e| e["src"] }
|
|
39
42
|
.compact
|
|
40
43
|
.select { |link| link.include? file_type }
|
|
41
|
-
.
|
|
44
|
+
.map { |link| add_base_url(link) }
|
|
42
45
|
end
|
|
43
46
|
|
|
44
47
|
def parse_html(raw_html)
|
|
45
48
|
Nokogiri::HTML(raw_html)
|
|
46
49
|
end
|
|
50
|
+
|
|
51
|
+
def add_base_url(str)
|
|
52
|
+
url_breakdown = url.split('/')
|
|
53
|
+
|
|
54
|
+
if url_breakdown.first.include?('https')
|
|
55
|
+
url_breakdown.third
|
|
56
|
+
else
|
|
57
|
+
url_breakdown.first
|
|
58
|
+
end
|
|
59
|
+
end
|
|
47
60
|
end
|
|
48
61
|
end
|
|
49
62
|
end
|