pulse-downloader 0.1.28 → 0.1.32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +0 -3
  3. data/Gemfile.lock +17 -17
  4. data/lib/pulse/downloader/client.rb +1 -1
  5. data/lib/pulse/downloader/version.rb +1 -1
  6. data/lib/pulse/downloader/web_page_parser.rb +16 -3
  7. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2a9e6c9185000eb45fe03fcf5ec6739aa0b4df48cd9733654cc2f312ca156b54
4
- data.tar.gz: f03dfbbbecbd75ffe795b194c0ac19689021e1a74a4668ffb6573b6193eb0627
3
+ metadata.gz: 0e6a15b4479cbf7318be107f94f57eba9a19037e76c26d2408853a6407c50008
4
+ data.tar.gz: 5e838d02dd9bd81290db3dbf4bc5a3b5427361953458dc026fb3db3db6a42e05
5
5
  SHA512:
6
- metadata.gz: 706da41a31ad0fda7b4a17b748f8271f758973510039d2e3eac79115761ef71e970eee042d3ef30ed2a080ae7e64f49584d8110c249022a4bca3029c48cc1024
7
- data.tar.gz: 97c3c92845700c69343eba19fb93a9b9de71fe44ceca0ff986e25df8cc69ffb71a48c778eaa145df4ae802295b148feaee768814c52124e4ca74079e8ca9e807
6
+ metadata.gz: e102a711d68eeef3f8a070c1668df9469bd5961f4820e4dd6b326c005ba34e3687373a58ddb057ae649eefe77c40181788ad6fa83f9c06b0bcd4eb0e11424855
7
+ data.tar.gz: b9dbd2385e65812d09a5a6ceeab9e31dc5457c69384d1b96e34bca5ea54cd3515201ec34b570fad6ae9f7717a63d039f6b4a853e8df1ef7336fb0f3bf4421c47
data/Gemfile CHANGED
@@ -2,6 +2,3 @@ source "https://rubygems.org"
2
2
 
3
3
  # Specify your gem's dependencies in pulse-downloader.gemspec
4
4
  gemspec
5
-
6
- gem "rake", "~> 12.0"
7
- gem "minitest", "~> 5.0"
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pulse-downloader (0.1.26)
4
+ pulse-downloader (0.1.31)
5
5
  active_attr (~> 0.15)
6
6
  httparty (~> 0.18)
7
7
  nokogiri (~> 1.11)
@@ -10,15 +10,15 @@ PATH
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- actionpack (6.1.3.2)
14
- actionview (= 6.1.3.2)
15
- activesupport (= 6.1.3.2)
13
+ actionpack (6.1.4.1)
14
+ actionview (= 6.1.4.1)
15
+ activesupport (= 6.1.4.1)
16
16
  rack (~> 2.0, >= 2.0.9)
17
17
  rack-test (>= 0.6.3)
18
18
  rails-dom-testing (~> 2.0)
19
19
  rails-html-sanitizer (~> 1.0, >= 1.2.0)
20
- actionview (6.1.3.2)
21
- activesupport (= 6.1.3.2)
20
+ actionview (6.1.4.1)
21
+ activesupport (= 6.1.4.1)
22
22
  builder (~> 3.1)
23
23
  erubi (~> 1.4)
24
24
  rails-dom-testing (~> 2.0)
@@ -27,20 +27,20 @@ GEM
27
27
  actionpack (>= 3.0.2, < 7.0)
28
28
  activemodel (>= 3.0.2, < 7.0)
29
29
  activesupport (>= 3.0.2, < 7.0)
30
- activemodel (6.1.3.2)
31
- activesupport (= 6.1.3.2)
32
- activesupport (6.1.3.2)
30
+ activemodel (6.1.4.1)
31
+ activesupport (= 6.1.4.1)
32
+ activesupport (6.1.4.1)
33
33
  concurrent-ruby (~> 1.0, >= 1.0.2)
34
34
  i18n (>= 1.6, < 2)
35
35
  minitest (>= 5.1)
36
36
  tzinfo (~> 2.0)
37
37
  zeitwerk (~> 2.3)
38
- addressable (2.7.0)
38
+ addressable (2.8.0)
39
39
  public_suffix (>= 2.0.2, < 5.0)
40
40
  ansi (1.5.0)
41
41
  builder (3.2.4)
42
42
  coderay (1.1.3)
43
- concurrent-ruby (1.1.8)
43
+ concurrent-ruby (1.1.9)
44
44
  crack (0.4.5)
45
45
  rexml
46
46
  crass (1.0.6)
@@ -52,13 +52,13 @@ GEM
52
52
  multi_xml (>= 0.5.2)
53
53
  i18n (1.8.10)
54
54
  concurrent-ruby (~> 1.0)
55
- loofah (2.9.1)
55
+ loofah (2.12.0)
56
56
  crass (~> 1.0.2)
57
57
  nokogiri (>= 1.5.9)
58
58
  method_source (1.0.0)
59
59
  mime-types (3.3.1)
60
60
  mime-types-data (~> 3.2015)
61
- mime-types-data (3.2021.0225)
61
+ mime-types-data (3.2021.0901)
62
62
  minitest (5.14.4)
63
63
  minitest-focus (1.1.2)
64
64
  minitest (>= 4, < 6)
@@ -69,7 +69,7 @@ GEM
69
69
  ruby-progressbar
70
70
  mocha (1.11.2)
71
71
  multi_xml (0.6.0)
72
- nokogiri (1.11.5-x86_64-linux)
72
+ nokogiri (1.12.4-x86_64-linux)
73
73
  racc (~> 1.4)
74
74
  options (2.3.2)
75
75
  progress_bar (1.3.3)
@@ -86,9 +86,9 @@ GEM
86
86
  rails-dom-testing (2.0.3)
87
87
  activesupport (>= 4.2.0)
88
88
  nokogiri (>= 1.6)
89
- rails-html-sanitizer (1.3.0)
89
+ rails-html-sanitizer (1.4.2)
90
90
  loofah (~> 2.3)
91
- rake (12.3.3)
91
+ rake (13.0.6)
92
92
  rexml (3.2.5)
93
93
  ruby-progressbar (1.11.0)
94
94
  timecop (0.9.4)
@@ -111,7 +111,7 @@ DEPENDENCIES
111
111
  mocha (~> 1.11.2)
112
112
  pry (~> 0.13)
113
113
  pulse-downloader!
114
- rake (~> 12.0)
114
+ rake (~> 13.0)
115
115
  timecop (~> 0.9.1)
116
116
  webmock (~> 3.8.3)
117
117
 
data/lib/pulse/downloader/client.rb CHANGED
@@ -71,7 +71,7 @@ module Pulse
71
71
  end
72
72
 
73
73
  fetch_file_paths.map do |file_path|
74
- download(file_path, @progress_bar)
74
+ download(file_path, @progress_bar) if save_data
75
75
  @progress_bar.increment!
76
76
  end
77
77
  end
data/lib/pulse/downloader/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Pulse
2
2
  module Downloader
3
- VERSION = "0.1.28"
3
+ VERSION = "0.1.32"
4
4
  end
5
5
  end
data/lib/pulse/downloader/web_page_parser.rb CHANGED
@@ -26,7 +26,7 @@ module Pulse
26
26
  def extract_file_urls(response, custom_path_root, type)
27
27
  return [] if response.body.nil? || response.body.empty?
28
28
 
29
- remove_base64(
29
+ remove_artefacts(
30
30
  extract_all_urls(response, custom_path_root, type) +
31
31
  extract_download_links(response, type) +
32
32
  extract_embedded_images(response, type)
@@ -40,7 +40,7 @@ module Pulse
40
40
  .find_all { |u| u =~ /^https?:/ }
41
41
  .compact
42
42
  .select { |link| (link.include? type || link.include?(custom_path_root)) }
43
- .map { |link| add_base_url(link) }
43
+ .map { |link| add_base_url(link, custom_path_root) }
44
44
  end
45
45
 
46
46
  def extract_download_links(response, type)
@@ -65,6 +65,17 @@ module Pulse
65
65
  .map { |link| add_base_url(link) }
66
66
  end
67
67
 
68
+ def remove_artefacts(urls)
69
+ urls = remove_extra_escape_characters(urls)
70
+ remove_base64(urls)
71
+ end
72
+
73
+ def remove_extra_escape_characters(urls)
74
+ urls.map do |url|
75
+ url.gsub("\">", '')
76
+ end
77
+ end
78
+
68
79
  def remove_base64(urls)
69
80
  urls.reject do |url|
70
81
  url.include?(':image/') || url.include?('base64')
@@ -75,7 +86,9 @@ module Pulse
75
86
  Nokogiri::HTML(raw_html)
76
87
  end
77
88
 
78
- def add_base_url(str)
89
+ def add_base_url(str, custom_path_root=nil)
90
+ return str if custom_path_root
91
+
79
92
  if !str.include?('https://') && !str.include?(base_url)
80
93
  "https://#{base_url}#{str}"
81
94
  else
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pulse-downloader
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.28
4
+ version: 0.1.32
5
5
  platform: ruby
6
6
  authors:
7
7
  - trex22
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-05-24 00:00:00.000000000 Z
11
+ date: 2021-09-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: httparty