pulse-downloader 0.1.14 → 0.1.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pulse/downloader/client.rb +3 -0
- data/lib/pulse/downloader/version.rb +1 -1
- data/lib/pulse/downloader/web_page_parser.rb +13 -0
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4d2a34a08bede3b49a3d3629fc2d785d6337dc62702dbeb86742323b4baa11aa
|
|
4
|
+
data.tar.gz: 29e276ed758b83ff22453f7774afd4e28aec0b6eac3436302287e239e73e0d73
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ddd1e433cc5be43243d83867107089945aa756328fa3e3fb4d239b682bfa88c29563e8401a3dccadd266d87169129ae994132a178179740c91520b790a7209c8
|
|
7
|
+
data.tar.gz: cb3c310e893e3bb8c095d1e42b299f5bb3c9f95128b1657f65e066e7c60d154b22947f5d70d8e675f71438e7b0fe5c83cecef5a8806ba8b1a08e29f4a8f4bfb9
|
|
@@ -8,6 +8,7 @@ module Pulse
|
|
|
8
8
|
|
|
9
9
|
attr_reader :url,
|
|
10
10
|
:file_type,
|
|
11
|
+
:scrape_images,
|
|
11
12
|
:save_data,
|
|
12
13
|
:save_path,
|
|
13
14
|
:read_from_save_path,
|
|
@@ -29,6 +30,7 @@ module Pulse
|
|
|
29
30
|
# TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
|
|
30
31
|
def initialize(url:,
|
|
31
32
|
file_type:,
|
|
33
|
+
scrape_images: false,
|
|
32
34
|
save_data: false,
|
|
33
35
|
save_path: '',
|
|
34
36
|
read_from_save_path: false,
|
|
@@ -40,6 +42,7 @@ module Pulse
|
|
|
40
42
|
|
|
41
43
|
@url = url
|
|
42
44
|
@file_type = file_type
|
|
45
|
+
@scrape_images = scrape_images
|
|
43
46
|
@save_data = save_data
|
|
44
47
|
@save_path = save_path
|
|
45
48
|
@read_from_save_path = read_from_save_path
|
|
@@ -19,7 +19,10 @@ module Pulse
|
|
|
19
19
|
|
|
20
20
|
def extract_file_urls(response)
|
|
21
21
|
return [] if response.body.nil? || response.body.empty?
|
|
22
|
+
extract_download_links(response) + extract_embedded_images(response)
|
|
23
|
+
end
|
|
22
24
|
|
|
25
|
+
def extract_download_links(response)
|
|
23
26
|
parse_html(response.body)
|
|
24
27
|
.css('a')
|
|
25
28
|
.to_a
|
|
@@ -28,6 +31,16 @@ module Pulse
|
|
|
28
31
|
.select { |link| link.include? file_type }
|
|
29
32
|
end
|
|
30
33
|
|
|
34
|
+
def extract_embedded_images(response)
|
|
35
|
+
parse_html(response.body)
|
|
36
|
+
.css('img')
|
|
37
|
+
.to_a
|
|
38
|
+
.map { |e| e["src"] }
|
|
39
|
+
.compact
|
|
40
|
+
.select { |link| link.include? file_type }
|
|
41
|
+
.select { |link| link.include? "https://" }
|
|
42
|
+
end
|
|
43
|
+
|
|
31
44
|
def parse_html(raw_html)
|
|
32
45
|
Nokogiri::HTML(raw_html)
|
|
33
46
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pulse-downloader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.15
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- trex22
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-05-
|
|
11
|
+
date: 2021-05-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: httparty
|