torrentify 0.3 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +6 -0
- data/Rakefile +1 -1
- data/lib/manager/imdb_manager.rb +30 -0
- data/lib/manager/mechanize_manager.rb +30 -3
- data/lib/manager/sites/extratorrent_parser.rb +64 -0
- data/lib/manager/sites/isohunt_parser.rb +61 -0
- data/lib/manager/sites/kickass_parser.rb +9 -7
- data/lib/manager/sites/piratebay_parser.rb +66 -0
- data/lib/model/torrent_model.rb +25 -5
- data/lib/torrentify.rb +11 -1
- data/test/manager/imdb_manager_test.rb +25 -0
- data/test/manager/sites/extratorrent_parser_test.rb +26 -0
- data/test/manager/sites/isohunt_parser_test.rb +26 -0
- data/test/manager/sites/kickass_parser_test.rb +14 -20
- data/test/manager/sites/piratebay_parser_test.rb +26 -0
- data/test/manager/sites/test_data/extratorrent_mock_response.html +281 -0
- data/test/manager/sites/test_data/isohunt_mock_response.html +456 -0
- data/test/manager/sites/test_data/kickass_mock_response.html +841 -0
- data/test/manager/sites/test_data/piratebay_mock_response.html +302 -0
- data/test/manager/test_data/imdb_mock_response.html +1636 -0
- data/test/model/torrent_model_test.rb +39 -0
- data/test/test_helper.rb +2 -0
- data/torrentify.gemspec +5 -4
- metadata +44 -10
- data/test/manager/mechanize_manager_test.rb +0 -55
- data/test/torrentify_test.rb +0 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c0fe80eda52f9b9b490ad4f0b49f2611aab3af2b
|
4
|
+
data.tar.gz: 705238ee08b81f6a69e722f65be498462eefd7d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6fa0f4fd05e0bb4ed3a4f72dd882728269acd5811ee21cac593018b5bf37cf7ed9a803cb2122dc402c4242173f90e6deb2264f23e504b2b706f7c75b6913632c
|
7
|
+
data.tar.gz: 886271bdbe42b40d064f1b85185f618623633b7ab64ab47b344ce9554f1086fcff00245f173ef05dbce36fe213aa55b3bbd90b3d9af18e8dcedc8187423283fb
|
data/.rubocop.yml
CHANGED
data/Rakefile
CHANGED
@@ -23,7 +23,7 @@ desc 'Run Unittests'
|
|
23
23
|
Rake::TestTask.new(:unitTest) do |t|
|
24
24
|
t.libs << 'test'
|
25
25
|
t.verbose = false
|
26
|
-
t.test_files = FileList['test
|
26
|
+
t.test_files = FileList['test/**/*_test.rb']
|
27
27
|
end
|
28
28
|
|
29
29
|
desc 'Run codeclimate - Sends coverage info to CodeClimate when in CI'
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
require 'mechanize'
|
3
|
+
|
4
|
+
# Class responsible for connecting to imdb
|
5
|
+
# e.g getting user watchlist
|
6
|
+
class ImdbManager
|
7
|
+
# Module responsible for setting up mechanize-agent
|
8
|
+
module Agent
|
9
|
+
def self.get_web_page(url)
|
10
|
+
agent = Mechanize.new
|
11
|
+
# Default File implementation does not work
|
12
|
+
agent.pluggable_parser.default = Mechanize::Page
|
13
|
+
agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
14
|
+
agent.get(url)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_watchlist(userid)
|
19
|
+
rss_url = 'http://rss.imdb.com/user/userid/watchlist'
|
20
|
+
url = rss_url.gsub('userid', userid)
|
21
|
+
page = Agent.get_web_page(url)
|
22
|
+
items = page.search('.//item')
|
23
|
+
titles = []
|
24
|
+
items.each do |item|
|
25
|
+
title = item.search('.//title')[0].content
|
26
|
+
titles.push(title)
|
27
|
+
end
|
28
|
+
titles
|
29
|
+
end
|
30
|
+
end
|
@@ -2,6 +2,9 @@
|
|
2
2
|
require 'rubygems'
|
3
3
|
require 'mechanize'
|
4
4
|
require_relative 'sites/kickass_parser'
|
5
|
+
require_relative 'sites/piratebay_parser'
|
6
|
+
require_relative 'sites/isohunt_parser'
|
7
|
+
require_relative 'sites/extratorrent_parser'
|
5
8
|
|
6
9
|
# Manager responsible for scraping webpagem
|
7
10
|
class MechanizeManager
|
@@ -12,6 +15,7 @@ class MechanizeManager
|
|
12
15
|
module Agent
|
13
16
|
def self.get_web_page(url)
|
14
17
|
agent = Mechanize.new
|
18
|
+
agent.pluggable_parser.default = Mechanize::Page
|
15
19
|
agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
16
20
|
agent.get(url)
|
17
21
|
end
|
@@ -19,10 +23,33 @@ class MechanizeManager
|
|
19
23
|
|
20
24
|
def search_kickass(search_term)
|
21
25
|
white_space = '%20'
|
22
|
-
|
23
|
-
|
26
|
+
kickass_url = 'https://kat.cr/usearch/'
|
27
|
+
url = kickass_url << search_term.gsub(' ', white_space)
|
24
28
|
page = Agent.get_web_page(url)
|
25
29
|
KickassParser.new(page).main_divs
|
26
|
-
|
30
|
+
end
|
31
|
+
|
32
|
+
def search_piratebay(search_term)
|
33
|
+
white_space = '%20'
|
34
|
+
pirate_url = 'https://thepiratebay.mn/search/'
|
35
|
+
url = pirate_url << search_term.gsub(' ', white_space)
|
36
|
+
page = Agent.get_web_page(url)
|
37
|
+
PirateBayParser.new(page).main_divs
|
38
|
+
end
|
39
|
+
|
40
|
+
def search_isohunt(search_term)
|
41
|
+
white_space = '%20'
|
42
|
+
isohunt_url = 'https://isohunt.to/torrents/?ihq='
|
43
|
+
url = isohunt_url << search_term.gsub(' ', white_space)
|
44
|
+
page = Agent.get_web_page(url)
|
45
|
+
IsohuntParser.new(page).main_divs
|
46
|
+
end
|
47
|
+
|
48
|
+
def search_extratorrent(search_term)
|
49
|
+
white_space = '+'
|
50
|
+
extratorrent_url = 'http://extratorrent.cc/search/?search='
|
51
|
+
url = extratorrent_url << search_term.gsub(' ', white_space)
|
52
|
+
page = Agent.get_web_page(url)
|
53
|
+
ExtratorrentParser.new(page).main_divs
|
27
54
|
end
|
28
55
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
require_relative '../../model/torrent_model'
|
4
|
+
|
5
|
+
# Class responsible for parsing
|
6
|
+
# the page response from kickass
|
7
|
+
class ExtratorrentParser
|
8
|
+
def initialize(page)
|
9
|
+
@page = page
|
10
|
+
end
|
11
|
+
|
12
|
+
# Parse values from html
|
13
|
+
module Parser
|
14
|
+
def self.seeders(div)
|
15
|
+
links = div.search(".//td[@class='sy']")
|
16
|
+
value = ''
|
17
|
+
links.each do |link|
|
18
|
+
value = link.content
|
19
|
+
end
|
20
|
+
value
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.leechers(div)
|
24
|
+
links = div.search(".//td[@class='ly']")
|
25
|
+
value = ''
|
26
|
+
links.each do |link|
|
27
|
+
value = link.content
|
28
|
+
end
|
29
|
+
value
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.size(div)
|
33
|
+
links = div.search('.//td')
|
34
|
+
links[3].content
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.torrent_url(div)
|
38
|
+
links = div.search('.//a')
|
39
|
+
links[0].attributes['href']
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.torrent_name(div)
|
43
|
+
links = div.search('.//a')
|
44
|
+
# Get link title and strip Download och torrent from result
|
45
|
+
title = links[0].attributes['title'].text
|
46
|
+
title.gsub!('Download ', '').gsub!(' torrent', '')
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def main_divs
|
51
|
+
divs = @page.search(".//tr[@class='tlr' or @class='tlz']")
|
52
|
+
torrents = []
|
53
|
+
divs.each do |div|
|
54
|
+
torrent_file = Parser.torrent_url(div)
|
55
|
+
name = Parser.torrent_name(div)
|
56
|
+
size = Parser.size(div)
|
57
|
+
seeders = Parser.seeders(div)
|
58
|
+
leechers = Parser.leechers(div)
|
59
|
+
torrent = Torrent.new(torrent_file, name, size, seeders, leechers)
|
60
|
+
torrents.push(torrent)
|
61
|
+
end
|
62
|
+
torrents
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
require_relative '../../model/torrent_model'
|
4
|
+
|
5
|
+
# Class responsible for parsing
|
6
|
+
# the page response from isohunt
|
7
|
+
class IsohuntParser
|
8
|
+
def initialize(page)
|
9
|
+
@page = page
|
10
|
+
end
|
11
|
+
|
12
|
+
# Parse values from html
|
13
|
+
module Parser
|
14
|
+
def self.seeders(div)
|
15
|
+
links = div.search(".//td[@class=' sy' or @class=' sn']")
|
16
|
+
value = ''
|
17
|
+
links.each do |link|
|
18
|
+
value = link.content
|
19
|
+
end
|
20
|
+
value
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.leechers(_div)
|
24
|
+
'N/A'
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.size(div)
|
28
|
+
links = div.search(".//td[@class='size-row']")
|
29
|
+
value = ''
|
30
|
+
links.each do |link|
|
31
|
+
value = link.content
|
32
|
+
end
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.torrent_url(div)
|
37
|
+
links = div.search('.//a[@href]')
|
38
|
+
links[0].attributes['href']
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.torrent_name(div)
|
42
|
+
links = div.search('.//a[@href]')
|
43
|
+
links[0].content
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def main_divs
|
48
|
+
divs = @page.search('.//tr[@data-key]')
|
49
|
+
torrents = []
|
50
|
+
divs.each do |div|
|
51
|
+
torrent_file = Parser.torrent_url(div)
|
52
|
+
name = Parser.torrent_name(div)
|
53
|
+
size = Parser.size(div)
|
54
|
+
seeders = Parser.seeders(div)
|
55
|
+
leechers = Parser.leechers(div)
|
56
|
+
torrent = Torrent.new(torrent_file, name, size, seeders, leechers)
|
57
|
+
torrents.push(torrent)
|
58
|
+
end
|
59
|
+
torrents
|
60
|
+
end
|
61
|
+
end
|
@@ -59,14 +59,16 @@ class KickassParser
|
|
59
59
|
|
60
60
|
def main_divs
|
61
61
|
divs = @page.search(".//tr[@class='even' or @class='odd']")
|
62
|
+
torrents = []
|
62
63
|
divs.each do |div|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
64
|
+
torrent_file = Parser.torrent_url(div)
|
65
|
+
name = Parser.torrent_name(div)
|
66
|
+
size = Parser.size(div)
|
67
|
+
seeders = Parser.seeders(div)
|
68
|
+
leechers = Parser.leechers(div)
|
69
|
+
torrent = Torrent.new(torrent_file, name, size, seeders, leechers)
|
70
|
+
torrents.push(torrent)
|
69
71
|
end
|
70
|
-
|
72
|
+
torrents
|
71
73
|
end
|
72
74
|
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
require_relative '../../model/torrent_model'
|
4
|
+
|
5
|
+
# Class responsible for parsing
|
6
|
+
# the page response from kickass
|
7
|
+
class PirateBayParser
|
8
|
+
def initialize(page)
|
9
|
+
@page = page
|
10
|
+
end
|
11
|
+
|
12
|
+
# Parse values from html
|
13
|
+
module Parser
|
14
|
+
def self.seeders(div)
|
15
|
+
links = div.search(".//td[@align='right']")
|
16
|
+
links[0].content if !links.nil? && !links[0].nil?
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.leechers(div)
|
20
|
+
links = div.search(".//td[@align='right']")
|
21
|
+
links[1].content if !links.nil? && !links[1].nil?
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.size(div)
|
25
|
+
links = div.search(".//font[@class='detDesc']")
|
26
|
+
value = ''
|
27
|
+
links.each do |link|
|
28
|
+
value = link.content[/#{'Size '}(.*?)#{', '}/m, 1]
|
29
|
+
end
|
30
|
+
value
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.torrent_url(div)
|
34
|
+
links = div.search(".//a[@title='Download this torrent using magnet']")
|
35
|
+
value = ''
|
36
|
+
links.each do |link|
|
37
|
+
value = link.attributes['href']
|
38
|
+
end
|
39
|
+
value
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.torrent_name(div)
|
43
|
+
links = div.search(".//a[@class='detLink']")
|
44
|
+
value = ''
|
45
|
+
links.each do |link|
|
46
|
+
value = link.content
|
47
|
+
end
|
48
|
+
value
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def main_divs
|
53
|
+
divs = @page.search('.//tr')
|
54
|
+
torrents = []
|
55
|
+
divs.each do |div|
|
56
|
+
torrent_file = Parser.torrent_url(div)
|
57
|
+
name = Parser.torrent_name(div)
|
58
|
+
size = Parser.size(div)
|
59
|
+
seeders = Parser.seeders(div)
|
60
|
+
leechers = Parser.leechers(div)
|
61
|
+
torrent = Torrent.new(torrent_file, name, size, seeders, leechers)
|
62
|
+
torrents.push(torrent)
|
63
|
+
end
|
64
|
+
torrents
|
65
|
+
end
|
66
|
+
end
|
data/lib/model/torrent_model.rb
CHANGED
@@ -2,9 +2,29 @@
|
|
2
2
|
|
3
3
|
# Model for saving torrent info
|
4
4
|
class Torrent
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
5
|
+
def initialize(torrent_file, name, size, seeders, leechers)
|
6
|
+
@torrent_file = torrent_file
|
7
|
+
@name = name
|
8
|
+
@size = size
|
9
|
+
@seeders = seeders
|
10
|
+
@leechers = leechers
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_s
|
14
|
+
string = StringIO.open do |s|
|
15
|
+
s.puts 'torrent_file: ' + @torrent_file.to_s
|
16
|
+
s.puts 'name: ' + @name.to_s
|
17
|
+
s.puts 'size: ' + @size.to_s
|
18
|
+
s.puts 'seeders: ' + @seeders.to_s
|
19
|
+
s.puts 'leechers: ' + @leechers.to_s
|
20
|
+
s.string
|
21
|
+
end
|
22
|
+
string
|
23
|
+
end
|
24
|
+
|
25
|
+
attr_reader :torrent_file
|
26
|
+
attr_reader :name
|
27
|
+
attr_reader :size
|
28
|
+
attr_reader :seeders
|
29
|
+
attr_reader :leechers
|
10
30
|
end
|
data/lib/torrentify.rb
CHANGED
@@ -1,11 +1,21 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
|
3
3
|
require_relative 'manager/mechanize_manager'
|
4
|
+
require_relative 'manager/imdb_manager'
|
4
5
|
|
5
6
|
# Main interface
|
6
7
|
# Responsible for running manager-methods
|
7
8
|
module Torrentify
|
8
9
|
def self.search(search_param)
|
9
|
-
MechanizeManager.new
|
10
|
+
manager = MechanizeManager.new
|
11
|
+
kickass = manager.search_kickass(search_param)
|
12
|
+
piratebay = manager.search_piratebay(search_param)
|
13
|
+
isohunt = manager.search_isohunt(search_param)
|
14
|
+
extratorrent = manager.search_extratorrent(search_param)
|
15
|
+
[kickass, piratebay, isohunt, extratorrent]
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.imdb_watchlist(userid)
|
19
|
+
ImdbManager.new.get_watchlist(userid)
|
10
20
|
end
|
11
21
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
require_relative '../../lib/manager/imdb_manager'
|
5
|
+
|
6
|
+
# Tests for kickass
|
7
|
+
class TestImdbClass < Test::Unit::TestCase
|
8
|
+
def test_get_watchlist
|
9
|
+
userid = 'ur32409321'
|
10
|
+
|
11
|
+
mock_request
|
12
|
+
|
13
|
+
result = ImdbManager.new.get_watchlist(userid)
|
14
|
+
|
15
|
+
assert_not_nil result
|
16
|
+
assert_false result.empty?
|
17
|
+
end
|
18
|
+
|
19
|
+
def mock_request
|
20
|
+
file_path = 'test_data/imdb_mock_response.html'
|
21
|
+
mock_data = File.read(File.join(__dir__, file_path))
|
22
|
+
WebMock.stub_request(:get, 'http://rss.imdb.com/user/ur32409321/watchlist')
|
23
|
+
.to_return(:status => 200, :body => mock_data, :headers => {})
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
require_relative '../../test_helper'
|
3
|
+
|
4
|
+
require_relative '../../../lib/manager/sites/extratorrent_parser'
|
5
|
+
require_relative '../../../lib/manager/mechanize_manager'
|
6
|
+
|
7
|
+
# Tests for kickass
|
8
|
+
class TestExtratorrentClass < Test::Unit::TestCase
|
9
|
+
def test_main_divs
|
10
|
+
search_term = 'a pigeon sat on a branch reflecting on existence'
|
11
|
+
|
12
|
+
mock_request
|
13
|
+
|
14
|
+
result = MechanizeManager.new.search_extratorrent(search_term)
|
15
|
+
|
16
|
+
assert_not_nil result
|
17
|
+
assert_false result.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def mock_request
|
21
|
+
file_path = 'test_data/extratorrent_mock_response.html'
|
22
|
+
mock_data = File.read(File.join(__dir__, file_path))
|
23
|
+
WebMock.stub_request(:get, 'http://extratorrent.cc/search/?search=a%20pigeon%20sat%20on%20a%20branch%20reflecting%20on%20existence')
|
24
|
+
.to_return(:status => 200, :body => mock_data, :headers => {})
|
25
|
+
end
|
26
|
+
end
|