prss 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/prss/downloader.rb +7 -19
- data/lib/prss/feed.rb +11 -5
- data/lib/prss/links.rb +9 -7
- data/lib/prss/version.rb +1 -1
- data/prss.gemspec +0 -2
- metadata +3 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 483d80cc3ae48fc4ce9c57a84373831b8692e4f6
|
4
|
+
data.tar.gz: 2ee7d8632383aec9cbe7b708998844f12b0aa4c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 91d05e06e171a1278b05c7fdceb42d24bb4252e21b8086f26f960e38ab9f895ecf946e3a61200e0dcfdd8b1b4224ffb8ba0eef1f0a33c907c7cba50f03af6fa1
|
7
|
+
data.tar.gz: af377731cdcb5d1f427d774e59b444f72c6afdaf9bb834cb07e8281a312f9a6aa68df497d38f1b1fa51cec74c111f2f3557105b4276f2cf72c82057a19609b33
|
data/lib/prss/downloader.rb
CHANGED
@@ -1,13 +1,11 @@
|
|
1
|
-
require '
|
1
|
+
require 'net/http'
|
2
2
|
require 'pathname'
|
3
3
|
|
4
4
|
module PRSS
|
5
5
|
class Downloader
|
6
|
-
attr_reader :hydra
|
7
6
|
|
8
7
|
def initialize(links)
|
9
8
|
@links = links
|
10
|
-
@hydra = Typhoeus::Hydra.new
|
11
9
|
end
|
12
10
|
|
13
11
|
def self.verify!(output_path)
|
@@ -19,30 +17,20 @@ module PRSS
|
|
19
17
|
def download_to(output_path)
|
20
18
|
output = self.class.verify!(output_path)
|
21
19
|
|
22
|
-
@files =
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
request.on_complete do |response|
|
27
|
-
@files << save_file(response, output)
|
28
|
-
end
|
29
|
-
|
30
|
-
hydra.queue request
|
31
|
-
end
|
32
|
-
|
33
|
-
hydra.run
|
34
|
-
|
35
|
-
@files
|
20
|
+
@files = @links.each.map do |uri|
|
21
|
+
response = Net::HTTP.get_response(uri)
|
22
|
+
save_file(response, output)
|
23
|
+
end.compact
|
36
24
|
end
|
37
25
|
|
38
26
|
def save_file(response, output)
|
39
|
-
filename = response
|
27
|
+
filename = response['Content-Disposition'][/filename="(.+)"$/ ,1]
|
40
28
|
file = output.join(filename)
|
41
29
|
|
42
30
|
return if file.exist?
|
43
31
|
|
44
32
|
open(file, 'wb') do |file|
|
45
|
-
|
33
|
+
response.read_body(file)
|
46
34
|
end
|
47
35
|
|
48
36
|
filename
|
data/lib/prss/feed.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
|
-
require '
|
1
|
+
require 'net/http'
|
2
2
|
require 'uri'
|
3
|
+
require 'benchmark'
|
3
4
|
|
4
5
|
module PRSS
|
5
6
|
class Feed
|
6
|
-
PERSONAL_FEED_URL =
|
7
|
+
PERSONAL_FEED_URL = ENV.fetch('PRSS_FEED_URL') { 'https://hdbits.org/rss/my'.freeze }
|
7
8
|
|
8
9
|
attr_reader :uri
|
9
10
|
|
@@ -11,16 +12,21 @@ module PRSS
|
|
11
12
|
#@passkey = passkey
|
12
13
|
@uri = URI(PERSONAL_FEED_URL)
|
13
14
|
@uri.query = "passkey=#{passkey}"
|
15
|
+
@uri.freeze
|
14
16
|
end
|
15
17
|
|
16
18
|
def response
|
17
|
-
|
18
|
-
|
19
|
+
response = nil
|
20
|
+
time = Benchmark.realtime do
|
21
|
+
response = Net::HTTP.get(@uri)
|
19
22
|
end
|
23
|
+
|
24
|
+
puts "Fetched #{@uri} in #{time.round(2)}s"
|
25
|
+
response
|
20
26
|
end
|
21
27
|
|
22
28
|
def output
|
23
|
-
response
|
29
|
+
response
|
24
30
|
end
|
25
31
|
|
26
32
|
def links
|
data/lib/prss/links.rb
CHANGED
@@ -1,24 +1,26 @@
|
|
1
1
|
require 'nokogiri'
|
2
|
+
require 'rexml/document'
|
3
|
+
require 'uri'
|
2
4
|
|
3
5
|
module PRSS
|
4
6
|
class Links
|
5
7
|
def initialize(xml)
|
6
|
-
@xml = xml
|
8
|
+
@xml = ::REXML::Document.new(xml)
|
7
9
|
end
|
8
10
|
|
9
|
-
|
10
|
-
@channel ||= Nokogiri::XML.parse(@xml).xpath('//channel')
|
11
|
-
end
|
11
|
+
attr_reader :xml
|
12
12
|
|
13
13
|
def links
|
14
|
-
|
14
|
+
xml.elements.to_a('//item/link').tap do |links|
|
15
15
|
puts "Found #{links.size} links"
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
def each
|
19
|
+
def each
|
20
|
+
return to_enum unless block_given?
|
21
|
+
|
20
22
|
links.each do |link|
|
21
|
-
yield link.
|
23
|
+
yield URI(link.text.strip)
|
22
24
|
end
|
23
25
|
end
|
24
26
|
|
data/lib/prss/version.rb
CHANGED
data/prss.gemspec
CHANGED
@@ -15,8 +15,6 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.require_paths = ["lib"]
|
16
16
|
gem.version = PRSS::VERSION
|
17
17
|
|
18
|
-
gem.add_dependency 'nokogiri', '~> 1.6.2'
|
19
|
-
gem.add_dependency 'typhoeus', '~> 0.6.9'
|
20
18
|
gem.add_dependency 'thor', '~> 0.19.1'
|
21
19
|
|
22
20
|
gem.add_development_dependency 'rspec', '~> 3.0'
|
metadata
CHANGED
@@ -1,43 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prss
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michal Cichra
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: nokogiri
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 1.6.2
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: 1.6.2
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: typhoeus
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 0.6.9
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: 0.6.9
|
41
13
|
- !ruby/object:Gem::Dependency
|
42
14
|
name: thor
|
43
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,11 +80,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
80
|
version: '0'
|
109
81
|
requirements: []
|
110
82
|
rubyforge_project:
|
111
|
-
rubygems_version: 2.
|
83
|
+
rubygems_version: 2.5.1
|
112
84
|
signing_key:
|
113
85
|
specification_version: 4
|
114
86
|
summary: Parses HDbits private RSS stream, downloads and saves all files to specified
|
115
87
|
directory
|
116
88
|
test_files:
|
117
89
|
- spec/spec_helper.rb
|
118
|
-
has_rdoc:
|