junkie 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.rspec +1 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/README.md +1 -1
- data/Rakefile +5 -0
- data/bin/junkie +10 -0
- data/junkie.gemspec +9 -0
- data/lib/junkie/config.rb +75 -0
- data/lib/junkie/episode.rb +30 -0
- data/lib/junkie/errors.rb +13 -0
- data/lib/junkie/helper.rb +13 -0
- data/lib/junkie/log.rb +16 -0
- data/lib/junkie/patched/sjunkieex.rb +79 -0
- data/lib/junkie/pyload/api.rb +161 -0
- data/lib/junkie/pyload/observer.rb +197 -0
- data/lib/junkie/reactor.rb +105 -0
- data/lib/junkie/version.rb +1 -1
- data/lib/junkie.rb +21 -2
- data/spec/config_spec.rb +39 -0
- data/spec/environment_spec.rb +7 -0
- data/spec/pyload_api_spec.rb +118 -0
- data/spec/spec_helper.rb +27 -0
- metadata +171 -5
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
data/Rakefile
CHANGED
data/bin/junkie
ADDED
data/junkie.gemspec
CHANGED
@@ -16,4 +16,13 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
17
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
18
|
gem.require_paths = ["lib"]
|
19
|
+
gem.add_runtime_dependency(%q<eventmachine>, [">= 1.0"])
|
20
|
+
gem.add_runtime_dependency(%q<em-http-request>, [">= 1.0"])
|
21
|
+
gem.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.1.0"])
|
22
|
+
gem.add_runtime_dependency(%q<nokogiri>, [">= 1.5"])
|
23
|
+
gem.add_runtime_dependency(%q<hashconfig>, [">= 0.0.1"])
|
24
|
+
gem.add_runtime_dependency(%q<serienrenamer>, [">= 0.0.1"])
|
25
|
+
gem.add_runtime_dependency(%q<sjunkieex>, [">= 0.0.1"])
|
26
|
+
gem.add_runtime_dependency(%q<sindex>, [">= 0.0.1"])
|
27
|
+
gem.add_runtime_dependency(%q<logging>, ["~> 1.8.0"])
|
19
28
|
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'hashconfig'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module Junkie
|
5
|
+
|
6
|
+
# Module that encapsulates the handling for configuration into a module
|
7
|
+
# which can be included into any Class which includes a constant Hash named
|
8
|
+
# DEFAULT_CONFIG is returned by Config.get_config(self) merged with the
|
9
|
+
# version from the config file
|
10
|
+
#
|
11
|
+
# @example
|
12
|
+
# class Foo
|
13
|
+
# include Config
|
14
|
+
#
|
15
|
+
# DEFAULT_CONFIG = { refresh: 5 }
|
16
|
+
#
|
17
|
+
# def initialize
|
18
|
+
# # the following holds the merged version from DEFAULT_CONFIG
|
19
|
+
# @config = Config.get_config(self)
|
20
|
+
# end
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
module Config
|
24
|
+
@including_classes = []
|
25
|
+
@comlete_config = nil
|
26
|
+
|
27
|
+
# Callback method which is called when a class includes the module
|
28
|
+
#
|
29
|
+
# @param [Class] the class which includes the module
|
30
|
+
def self.included(mod)
|
31
|
+
@including_classes << mod
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
# Get the config hash for the calling module merged with the hash from the
|
36
|
+
# config file
|
37
|
+
#
|
38
|
+
# @param [Class] should be equal to `self`
|
39
|
+
# @return [Hash] merged version of DEFAULT_CONFIG from the calling class
|
40
|
+
def self.get_config(source)
|
41
|
+
|
42
|
+
# builds up the complete config and merges them with serialzed version
|
43
|
+
# at the first call to this method
|
44
|
+
if @comlete_config.nil?
|
45
|
+
default_config = self.collect_default_configs
|
46
|
+
|
47
|
+
FileUtils.mkdir_p File.dirname(Junkie::CONFIG_FILE)
|
48
|
+
@comlete_config =
|
49
|
+
default_config.merge_with_serialized(Junkie::CONFIG_FILE)
|
50
|
+
end
|
51
|
+
|
52
|
+
return @comlete_config[source.class.to_s]
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
# Collects the DEFAULT_CONFIG from all including Classes
|
57
|
+
#
|
58
|
+
# @return [Hash] hash of hashes indexed by class name
|
59
|
+
def self.collect_default_configs
|
60
|
+
config = Hash.new
|
61
|
+
|
62
|
+
@including_classes.each do |klass|
|
63
|
+
if not klass.const_defined? :DEFAULT_CONFIG
|
64
|
+
raise NotImplementedError,
|
65
|
+
"#{klass} has to include a constant DEFAULT_CONFIG as Hash"
|
66
|
+
end
|
67
|
+
|
68
|
+
config[klass.to_s] = klass.const_get :DEFAULT_CONFIG
|
69
|
+
end
|
70
|
+
|
71
|
+
config
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
module Junkie
|
3
|
+
|
4
|
+
require 'sindex'
|
5
|
+
|
6
|
+
class Episode
|
7
|
+
attr_reader :id, :series, :found_at, :link
|
8
|
+
attr_accessor :description, :status
|
9
|
+
|
10
|
+
def initialize(series, link, description=nil)
|
11
|
+
@series = series
|
12
|
+
@link = link
|
13
|
+
@found_at = DateTime.now
|
14
|
+
@description = description
|
15
|
+
@status = :found
|
16
|
+
@id = episode_identifier
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
"%s (%s)" % [ @series, episode_identifier ]
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def episode_identifier
|
26
|
+
Sindex::SeriesIndex.extract_episode_identifier(@description)
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Junkie
|
4
|
+
class InvalidCredentialsError < Exception; end
|
5
|
+
|
6
|
+
class HTTP403Error < Exception; end
|
7
|
+
class HTTP404Error < Exception; end
|
8
|
+
class HTTP500Error < Exception; end
|
9
|
+
|
10
|
+
class InvalidConfigError < Exception; end
|
11
|
+
|
12
|
+
class InvalidStateError < Exception; end
|
13
|
+
end
|
data/lib/junkie/log.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'logging'
|
2
|
+
|
3
|
+
module Junkie
|
4
|
+
|
5
|
+
# Utility module that includes a logging function `log` into the class
|
6
|
+
# that includes this mdoule
|
7
|
+
module Log
|
8
|
+
|
9
|
+
# constructs a new Logger for the class
|
10
|
+
#
|
11
|
+
# @return [Logging.logger] returns a Logger instance for the current class
|
12
|
+
def log
|
13
|
+
@log ||= Logging.logger[self]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'em-http'
|
2
|
+
require 'fiber'
|
3
|
+
require 'junkie/log'
|
4
|
+
require 'sjunkieex'
|
5
|
+
|
6
|
+
module Junkie
|
7
|
+
|
8
|
+
# Monkex-patched version of the Interface from the `sjunkieex` gem
|
9
|
+
class ::Sjunkieex::Interface
|
10
|
+
include Junkie::Log
|
11
|
+
|
12
|
+
|
13
|
+
# Method that searches for new episodes and returns it in a nice structure
|
14
|
+
#
|
15
|
+
# @note should be called inside a Ruby fiber
|
16
|
+
# @return [Hash] series names as keys, data as values
|
17
|
+
def get_links_for_downloads
|
18
|
+
episodes = []
|
19
|
+
look_for_new_episodes.each do |link,series|
|
20
|
+
|
21
|
+
links = parse_series_page(series, link)
|
22
|
+
links.each do |identifier, link_data|
|
23
|
+
|
24
|
+
hd = @options[:hd_enabled]
|
25
|
+
|
26
|
+
# select links, depending on wanted resolution
|
27
|
+
links = []
|
28
|
+
if hd
|
29
|
+
if link_data[:hd_1080p]
|
30
|
+
links = link_data[:hd_1080p]
|
31
|
+
elsif link_data[:hd_720p]
|
32
|
+
links = link_data[:hd_720p]
|
33
|
+
end
|
34
|
+
else
|
35
|
+
(links = link_data[:sd]) if link_data[:sd]
|
36
|
+
end
|
37
|
+
|
38
|
+
if links.empty?
|
39
|
+
log.info("#{series}(#{identifier}) no links in this resolution")
|
40
|
+
next
|
41
|
+
end
|
42
|
+
|
43
|
+
download_links = links.select do |link|
|
44
|
+
link.match(/\/f-\w+\/#{ @options[:hoster_id] }_/)
|
45
|
+
end
|
46
|
+
|
47
|
+
if download_links.empty?
|
48
|
+
puts "there are no links for this hoster"
|
49
|
+
next
|
50
|
+
end
|
51
|
+
|
52
|
+
episode = Junkie::Episode.new(
|
53
|
+
series, download_links.first, link_data[:episodedata])
|
54
|
+
episodes << episode
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
episodes
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def get_page_data(link)
|
64
|
+
f = Fiber.current
|
65
|
+
http = EventMachine::HttpRequest.new(link).get :timeout => 10
|
66
|
+
|
67
|
+
http.callback { f.resume(http) }
|
68
|
+
http.errback { f.resume(http) }
|
69
|
+
|
70
|
+
Fiber.yield
|
71
|
+
|
72
|
+
if http.error
|
73
|
+
raise IOError, http.error
|
74
|
+
end
|
75
|
+
|
76
|
+
http.response
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,161 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'em-http'
|
3
|
+
require 'em-http/middleware/json_response'
|
4
|
+
require 'yaml'
|
5
|
+
require 'json'
|
6
|
+
require 'uri'
|
7
|
+
|
8
|
+
module Junkie
|
9
|
+
module Pyload
|
10
|
+
|
11
|
+
# Class that abstracts the Pyload Api
|
12
|
+
class Api
|
13
|
+
include Config
|
14
|
+
|
15
|
+
attr_reader :config, :cookie
|
16
|
+
|
17
|
+
FILE_STATUS = {
|
18
|
+
0 => :done,
|
19
|
+
2 => :online,
|
20
|
+
3 => :queued,
|
21
|
+
4 => :skipped,
|
22
|
+
5 => :waiting,
|
23
|
+
8 => :failed,
|
24
|
+
10 => :decrypting,
|
25
|
+
12 => :downloading,
|
26
|
+
13 => :extracting,
|
27
|
+
}
|
28
|
+
|
29
|
+
DEFAULT_CONFIG = {
|
30
|
+
protocol: 'http',
|
31
|
+
host: 'localhost',
|
32
|
+
port: 8000,
|
33
|
+
api_user: '',
|
34
|
+
api_password: ''
|
35
|
+
}
|
36
|
+
|
37
|
+
def initialize()
|
38
|
+
@config = Config.get_config(self)
|
39
|
+
|
40
|
+
api_user = @config[:api_user]
|
41
|
+
api_password = @config[:api_password]
|
42
|
+
|
43
|
+
unless api_password && api_user &&
|
44
|
+
api_password.match(/\w+/) && api_user.match(/\w+/)
|
45
|
+
raise InvalidConfigError, "api_user or api_password are not configured"
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
# makes a GET request with the given method name and params
|
51
|
+
#
|
52
|
+
# @param [Symbol] API-method e.g :getQueue, :getPackageData
|
53
|
+
# @param [Hash] params that are a passed as query parameters
|
54
|
+
#
|
55
|
+
# @return [Object] parsed JSON Response
|
56
|
+
def call(method, params={})
|
57
|
+
raise ArgumentError, "`method` is not a Symbol" unless method.is_a? Symbol
|
58
|
+
|
59
|
+
query_parameter = {}
|
60
|
+
params.each do |key, value|
|
61
|
+
if value.is_a? Array
|
62
|
+
query_parameter[key] =
|
63
|
+
URI.encode_www_form_component(JSON.dump(value))
|
64
|
+
else
|
65
|
+
query_parameter[key] =
|
66
|
+
URI.encode_www_form_component('"' + value.to_s + '"')
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
request("/#{method.to_s}", :get, query_parameter)
|
71
|
+
end
|
72
|
+
|
73
|
+
|
74
|
+
# Logs the api user in through sending the credentials to the Api
|
75
|
+
#
|
76
|
+
# @note sets cookie instance variable
|
77
|
+
# @raise [Junkie::InvalidCredentialsError] is raised when the Login fails
|
78
|
+
def login
|
79
|
+
resp = nil
|
80
|
+
|
81
|
+
begin
|
82
|
+
resp = request('/login', :post, {
|
83
|
+
username: @config[:api_user],
|
84
|
+
password: @config[:api_password]
|
85
|
+
}, true)
|
86
|
+
|
87
|
+
rescue Junkie::HTTP403Error => e
|
88
|
+
raise Junkie::InvalidCredentialsError,
|
89
|
+
"the supplied credentials are incorrect"
|
90
|
+
end
|
91
|
+
|
92
|
+
# extract Cookie-ID from Login response
|
93
|
+
if resp.response_header["SET_COOKIE"]
|
94
|
+
header = resp.response_header["SET_COOKIE"]
|
95
|
+
if md = header.match(/^(\S+);/)
|
96
|
+
@cookie = md[1]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# builds up a URL string
|
102
|
+
#
|
103
|
+
# @param [String] path the method specific part of the url
|
104
|
+
# @return [String] complete URI String
|
105
|
+
def build_url(path="")
|
106
|
+
"%s://%s:%d/api%s" %
|
107
|
+
[ @config[:protocol], @config[:host], @config[:port], path ]
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
|
113
|
+
# Utility function for making requests
|
114
|
+
#
|
115
|
+
# @param [String] method specific path e.g /getConfig or /login
|
116
|
+
# @param [Symbol] HTTP method that is issued :get, :post, :delete
|
117
|
+
# @param [Hash] Params that are added to the request, as query parameter
|
118
|
+
# on :get requests and body on :post requests
|
119
|
+
# @param [Boolean] return full response instead of body
|
120
|
+
#
|
121
|
+
# @return [Object] JSON Object that is send from the server or complete
|
122
|
+
# response if the previous argument is set
|
123
|
+
def request(path, method=:get, params={}, complete=false)
|
124
|
+
f = Fiber.current
|
125
|
+
|
126
|
+
request_options = {}
|
127
|
+
if method == :get
|
128
|
+
request_options[:query] = params
|
129
|
+
elsif method == :post
|
130
|
+
request_options[:body] = params
|
131
|
+
end
|
132
|
+
|
133
|
+
if @cookie
|
134
|
+
request_options[:head] = {'cookie' => @cookie}
|
135
|
+
end
|
136
|
+
|
137
|
+
conn = EventMachine::HttpRequest.new(build_url(path))
|
138
|
+
conn.use EventMachine::Middleware::JSONResponse
|
139
|
+
|
140
|
+
http = conn.method(method).call request_options
|
141
|
+
|
142
|
+
http.callback { f.resume(http) }
|
143
|
+
http.errback { f.resume(http) }
|
144
|
+
|
145
|
+
Fiber.yield
|
146
|
+
|
147
|
+
case http.response_header.status
|
148
|
+
when 403
|
149
|
+
raise Junkie::HTTP403Error, "Response has status code 403"
|
150
|
+
when 404
|
151
|
+
raise Junkie::HTTP404Error, "Response has status code 404"
|
152
|
+
when 500
|
153
|
+
raise Junkie::HTTP500Error, "Response has status code 505"
|
154
|
+
end
|
155
|
+
|
156
|
+
return http if complete
|
157
|
+
http.response
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,197 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'junkie/helper'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Junkie
|
6
|
+
module Pyload
|
7
|
+
class Observer
|
8
|
+
include Log, Helper, Config
|
9
|
+
|
10
|
+
DEFAULT_CONFIG = {
|
11
|
+
watchdog_refresh: 10, # interval the watchdog_timer is fired
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize(config={})
|
15
|
+
@config = Config.get_config(self)
|
16
|
+
|
17
|
+
@api = Junkie::Pyload::Api.new
|
18
|
+
|
19
|
+
@ready_for_new_links = true
|
20
|
+
@watchdog = nil
|
21
|
+
@active_downloads = Hash.new
|
22
|
+
end
|
23
|
+
|
24
|
+
# is a method that is called once from inside the reactor and allows us
|
25
|
+
# to register custom actions into the reactor
|
26
|
+
def setup
|
27
|
+
in_fiber {
|
28
|
+
@api.login
|
29
|
+
cleanup
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
# Adds new episode to Pyload which downloads the episode and extracts it
|
34
|
+
#
|
35
|
+
# @param [Junkie::Episode] episode which should be downloaded
|
36
|
+
#
|
37
|
+
# @note should only be called if `is_ready?` returns true
|
38
|
+
def add_episode(episode)
|
39
|
+
raise InvalidStateError, "Observer is not ready" unless is_ready?
|
40
|
+
@ready_for_new_links = false
|
41
|
+
|
42
|
+
episode.status = :downloading
|
43
|
+
|
44
|
+
package = "%s@%s" % [ episode.id, episode.series ]
|
45
|
+
|
46
|
+
pid = @api.call(:addPackage, {name: package, links: [episode.link]})
|
47
|
+
|
48
|
+
@active_downloads[pid] = episode
|
49
|
+
|
50
|
+
log.info("Added #{episode} to Pyload, Pid=#{pid}")
|
51
|
+
|
52
|
+
# start watchdog timer if it does not already run
|
53
|
+
if @watchdog.nil?
|
54
|
+
@watchdog = EM::PeriodicTimer.new(@config[:watchdog_refresh]) do
|
55
|
+
monitor_progress
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# checks if the Observer is ready to add new episodes to Pyload
|
61
|
+
#
|
62
|
+
# @returns [Boolean] true if new links can be added
|
63
|
+
def is_ready?
|
64
|
+
@ready_for_new_links
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
# This method is called from the watchdog timer periodically
|
70
|
+
#
|
71
|
+
# It monitors the download process and reacts depending on the results
|
72
|
+
def monitor_progress
|
73
|
+
log.debug("Watchdog timer has been fired")
|
74
|
+
|
75
|
+
in_fiber {
|
76
|
+
catch(:break) {
|
77
|
+
queue_data = @api.call(:getQueueData)
|
78
|
+
|
79
|
+
if queue_data.empty?
|
80
|
+
log.info("Empty Pyload queue, I cancel the watchdog timer")
|
81
|
+
unschedule_watchdog
|
82
|
+
throw :break
|
83
|
+
end
|
84
|
+
|
85
|
+
# extract package IDs and map them to current downloads
|
86
|
+
update_package_ids(queue_data)
|
87
|
+
|
88
|
+
if has_queue_any_failed_links?(queue_data)
|
89
|
+
log.info("There are failed links in the Queue, will fix this")
|
90
|
+
@api.call(:restartFailed)
|
91
|
+
throw :break
|
92
|
+
end
|
93
|
+
|
94
|
+
# look for complete downloads
|
95
|
+
pids = get_complete_downloads(queue_data)
|
96
|
+
|
97
|
+
if not pids.empty?
|
98
|
+
@api.call(:deletePackages, {pids: pids})
|
99
|
+
log.info("Complete packages are removed from Pyload Queue")
|
100
|
+
|
101
|
+
@ready_for_new_links = true
|
102
|
+
end
|
103
|
+
}
|
104
|
+
}
|
105
|
+
end
|
106
|
+
|
107
|
+
# Searches for failed links in the pyload queue
|
108
|
+
#
|
109
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
110
|
+
#
|
111
|
+
# @return [Boolean] true if there are any failed links, false otherwise
|
112
|
+
def has_queue_any_failed_links?(queue_data)
|
113
|
+
queue_data.each do |package|
|
114
|
+
package['links'].each do |link|
|
115
|
+
status = Pyload::Api::FILE_STATUS[link['status']]
|
116
|
+
return true if status == :failed
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
false
|
121
|
+
end
|
122
|
+
|
123
|
+
# Looks for complete downloads and returns their pids
|
124
|
+
#
|
125
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
126
|
+
#
|
127
|
+
# @return [Array] list of pids there the download is complete
|
128
|
+
def get_complete_downloads(queue_data)
|
129
|
+
pids = []
|
130
|
+
queue_data.each do |package|
|
131
|
+
next unless package['links'].size > 0
|
132
|
+
|
133
|
+
next if package['linksdone'] == 0
|
134
|
+
|
135
|
+
# When extracting is in progress the status of the first link is set
|
136
|
+
# to :extracting
|
137
|
+
extracting = package['links'].select do |link|
|
138
|
+
Pyload::Api::FILE_STATUS[link['status']] == :extracting
|
139
|
+
end
|
140
|
+
next unless extracting.empty?
|
141
|
+
|
142
|
+
sizetotal = package['sizetotal'].to_i
|
143
|
+
sizedone = package['sizedone'].to_i
|
144
|
+
|
145
|
+
if sizetotal > 0 && sizedone > 0
|
146
|
+
|
147
|
+
if sizetotal == sizedone
|
148
|
+
pids << package['pid']
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
pids
|
154
|
+
end
|
155
|
+
|
156
|
+
# extract package IDs and map them to current downloads
|
157
|
+
#
|
158
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
159
|
+
def update_package_ids(queue_data)
|
160
|
+
queue_data.each do |package|
|
161
|
+
pid = package['pid']
|
162
|
+
next if @active_downloads.has_key? pid
|
163
|
+
|
164
|
+
if @active_downloads.has_key? pid-1
|
165
|
+
log.info("Package ID has been changed, I will correct this")
|
166
|
+
@active_downloads[pid] = @active_downloads[pid-1]
|
167
|
+
@active_downloads.delete(pid-1)
|
168
|
+
next
|
169
|
+
end
|
170
|
+
|
171
|
+
raise InvalidStateError,
|
172
|
+
"PackageID #{pid} can't be mapped to active Download"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
# cancels the watchdog timer
|
177
|
+
def unschedule_watchdog
|
178
|
+
if @watchdog_timer
|
179
|
+
@watchdog_timer.cancel
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def cleanup
|
184
|
+
log.debug("Made a cleanup of Pyload's Queue")
|
185
|
+
@api.call(:stopAllDownloads)
|
186
|
+
|
187
|
+
packages = @api.call(:getQueue).map { |e| e['pid'] }
|
188
|
+
|
189
|
+
@api.call(:deletePackages, {pids: packages})
|
190
|
+
log.debug("The following packages have been removed: #{packages}")
|
191
|
+
|
192
|
+
@api.call(:unpauseServer)
|
193
|
+
end
|
194
|
+
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'sindex'
|
3
|
+
require 'sjunkieex'
|
4
|
+
require 'eventmachine'
|
5
|
+
require 'fiber'
|
6
|
+
require 'yaml'
|
7
|
+
require 'junkie'
|
8
|
+
require 'junkie/pyload/api'
|
9
|
+
require 'junkie/pyload/observer'
|
10
|
+
require 'junkie/errors'
|
11
|
+
|
12
|
+
module Junkie
|
13
|
+
|
14
|
+
class Reactor
|
15
|
+
include Log, Helper, Config
|
16
|
+
|
17
|
+
DEFAULT_CONFIG = {
|
18
|
+
:hd_enabled => false,
|
19
|
+
:hoster_id => "ul",
|
20
|
+
:series_index_file => File.join(Dir.home, '.sindex/seriesindex.xml'),
|
21
|
+
:episode_queue_timer_refresh => 5, # in seconds
|
22
|
+
:episode_search_refresh => 15, # in minutes
|
23
|
+
}
|
24
|
+
|
25
|
+
|
26
|
+
def initialize
|
27
|
+
@config = Config.get_config(self)
|
28
|
+
|
29
|
+
@sindex = Sindex::SeriesIndex.new(index_file: @config[:series_index_file])
|
30
|
+
@sjunkieex = Sjunkieex::Interface.new(@sindex, @config)
|
31
|
+
@pyload_observer = Junkie::Pyload::Observer.new()
|
32
|
+
|
33
|
+
@episode_queue = EM::Queue.new
|
34
|
+
@found_episodes = Hash.new
|
35
|
+
build_procs # has to be called here
|
36
|
+
end
|
37
|
+
|
38
|
+
def build_procs
|
39
|
+
|
40
|
+
# Proc that looks for new episodes on Seriesjunkies.org and adds them to
|
41
|
+
# the episode_queue if they are new
|
42
|
+
@look_for_new_episodes = Proc.new {
|
43
|
+
in_fiber {
|
44
|
+
log.info "Looking for new episodes"
|
45
|
+
|
46
|
+
@sjunkieex.get_links_for_downloads.each do |episode|
|
47
|
+
identifier = "%s@%s" % [episode.id, episode.series]
|
48
|
+
|
49
|
+
if not @found_episodes.has_key? identifier
|
50
|
+
log.info("Found new episode '#{episode}'")
|
51
|
+
@episode_queue.push(episode)
|
52
|
+
@found_episodes[identifier] = episode
|
53
|
+
end
|
54
|
+
end
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
# Proc that checks is Pyload-Observer is ready for new episodes and the
|
59
|
+
# episode_queue contains new episodes.
|
60
|
+
#
|
61
|
+
# @note Is called from within the reactor
|
62
|
+
@add_episodes_to_pyload = Proc.new do
|
63
|
+
if @pyload_observer.is_ready?
|
64
|
+
@episode_queue.pop do |episode|
|
65
|
+
log.info("Popped episode '#{episode}' from queue")
|
66
|
+
in_fiber {
|
67
|
+
@pyload_observer.add_episode(episode)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
###########################################################################
|
75
|
+
#################### The Reactor ##########################################
|
76
|
+
###########################################################################
|
77
|
+
def start
|
78
|
+
log.info("Starting Junkie #{Junkie::VERSION}")
|
79
|
+
|
80
|
+
EM.run do
|
81
|
+
|
82
|
+
# do some initialization work
|
83
|
+
@pyload_observer.setup
|
84
|
+
|
85
|
+
# Look for new episodes and add them to the Queue if they haven't
|
86
|
+
# been found yet
|
87
|
+
EM.next_tick do
|
88
|
+
@look_for_new_episodes.call
|
89
|
+
|
90
|
+
EM.add_periodic_timer(@config[:episode_search_refresh] * 60) do
|
91
|
+
@look_for_new_episodes.call
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# Add found episodes into Pyload if there are any episodes and pyload
|
96
|
+
# is ready
|
97
|
+
EM.add_periodic_timer(
|
98
|
+
@config[:episode_queue_timer_refresh], @add_episodes_to_pyload)
|
99
|
+
|
100
|
+
# for determining blocking operations
|
101
|
+
# EM.add_periodic_timer(1) { puts Time.now.to_i }
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/junkie/version.rb
CHANGED
data/lib/junkie.rb
CHANGED
@@ -1,5 +1,24 @@
|
|
1
|
-
require
|
1
|
+
require 'junkie/version'
|
2
|
+
require 'junkie/config'
|
3
|
+
require 'junkie/episode'
|
4
|
+
require 'junkie/log'
|
5
|
+
require 'junkie/reactor'
|
6
|
+
require 'junkie/errors'
|
7
|
+
require 'junkie/helper'
|
8
|
+
require 'junkie/patched/sjunkieex'
|
9
|
+
require 'junkie/pyload/api'
|
10
|
+
require 'junkie/pyload/observer'
|
11
|
+
require 'logging'
|
12
|
+
|
13
|
+
Logging.logger.root.appenders = Logging.appenders.stdout
|
14
|
+
Logging.logger.root.level = :debug
|
2
15
|
|
3
16
|
module Junkie
|
4
|
-
|
17
|
+
|
18
|
+
CONFIG_FILE = File.join(Dir.home, '.junkie', 'config.yml')
|
19
|
+
|
20
|
+
def start_reactor
|
21
|
+
r = Reactor.new
|
22
|
+
r.start
|
23
|
+
end
|
5
24
|
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
class ConfigTest
|
5
|
+
include Junkie::Config
|
6
|
+
|
7
|
+
attr_reader :config
|
8
|
+
|
9
|
+
DEFAULT_CONFIG = {
|
10
|
+
refresh: 5,
|
11
|
+
numbers: [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize()
|
15
|
+
@config = Junkie::Config.get_config(self)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe ConfigTest do
|
20
|
+
|
21
|
+
before do
|
22
|
+
@file = '/tmp/testtest.yml'
|
23
|
+
stub_const("Junkie::CONFIG_FILE", @file)
|
24
|
+
end
|
25
|
+
|
26
|
+
after(:each) do
|
27
|
+
FileUtils.rm(@file) if File.file? @file
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should stub out the Config File" do
|
31
|
+
expect(Junkie::CONFIG_FILE).to eq('/tmp/testtest.yml')
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should return the default if there are no changes made" do
|
35
|
+
test = ConfigTest.new
|
36
|
+
expect(test.config).to eq ConfigTest::DEFAULT_CONFIG
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
# WebMock.disable_net_connect!
|
5
|
+
|
6
|
+
describe Junkie::Pyload::Api do
|
7
|
+
include EMHelper
|
8
|
+
|
9
|
+
# remove the Config module behaviour
|
10
|
+
before(:each) do
|
11
|
+
Junkie::Config.stub!(:get_config).and_return(
|
12
|
+
Junkie::Pyload::Api::DEFAULT_CONFIG.merge(
|
13
|
+
{api_user: 'user', api_password: 'pass'}))
|
14
|
+
end
|
15
|
+
|
16
|
+
context "standard configuration" do
|
17
|
+
|
18
|
+
before(:each) do
|
19
|
+
@api = Junkie::Pyload::Api.new()
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should have the right standard url" do
|
23
|
+
expect(@api.build_url).to eq "http://localhost:8000/api"
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
context "#login" do
|
29
|
+
|
30
|
+
context "valid credentials" do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@api = Junkie::Pyload::Api.new()
|
34
|
+
@stub = stub_request(:post, "http://localhost:8000/api/login").
|
35
|
+
with(:body => "username=user&password=pass").
|
36
|
+
to_return(:status => 200, :body => "",
|
37
|
+
:headers => {SET_COOKIE: "api.id=123456; valid until it is removed"})
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should use the right url for API login" do
|
41
|
+
em {
|
42
|
+
@api.login
|
43
|
+
@stub.should have_been_requested
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should extract the Cookie from the login response" do
|
48
|
+
em {
|
49
|
+
@api.login
|
50
|
+
expect(@api.cookie).to eq "api.id=123456"
|
51
|
+
}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context "invalid credentials" do
|
56
|
+
|
57
|
+
before(:each) do
|
58
|
+
@api = Junkie::Pyload::Api.new()
|
59
|
+
@api.config[:api_password] = "invalid"
|
60
|
+
@stub = stub_request(:post, "http://localhost:8000/api/login").
|
61
|
+
with(:body => "username=user&password=invalid").
|
62
|
+
to_return(:status => 403, :body => "", :headers => {})
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should raise an error if login does not work" do
|
66
|
+
em {
|
67
|
+
expect {
|
68
|
+
@api.login
|
69
|
+
}.to raise_error(Junkie::InvalidCredentialsError)
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
context "#call" do
|
77
|
+
|
78
|
+
before(:each) do
|
79
|
+
@api = Junkie::Pyload::Api.new()
|
80
|
+
@api.stub(:cookie).and_return("api.id=123456")
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should call the right url for the method name" do
|
84
|
+
em {
|
85
|
+
stub = stub_request(:get, "http://localhost:8000/api/getQueue")
|
86
|
+
@api.call(:getQueue)
|
87
|
+
stub.should have_been_requested
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should pass parameters as JSON to the Request" do
|
92
|
+
em {
|
93
|
+
stub_request(:get, "http://localhost:8000/api/addPackage?links=%255B%2522http%253A%252F%252Ftest.test.de%2522%255D&name=%2522testpkg%2522").
|
94
|
+
to_return(:status => 200, :body => "76",
|
95
|
+
:headers => {"Content-Type" => "application/json"})
|
96
|
+
|
97
|
+
res = @api.call(:addPackage, { name: "testpkg", links: [ "http://test.test.de" ]})
|
98
|
+
res.should eq 76
|
99
|
+
}
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should build ruby datatypes from the JSON" do
|
103
|
+
em {
|
104
|
+
stub_request(:get, "http://localhost:8000/api/statusServer").
|
105
|
+
to_return(:status => 200, :body => '{ "pause": false, "total": 141 }',
|
106
|
+
:headers => {"Content-Type" => "application/json"})
|
107
|
+
|
108
|
+
res = @api.call(:statusServer)
|
109
|
+
hash = { "pause" => false, "total" => 141 }
|
110
|
+
res.should eq hash
|
111
|
+
}
|
112
|
+
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
end
|
118
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../lib/junkie'
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'rspec'
|
5
|
+
require 'eventmachine'
|
6
|
+
require 'fiber'
|
7
|
+
require 'webmock/rspec'
|
8
|
+
|
9
|
+
RSpec.configure do |config|
|
10
|
+
end
|
11
|
+
|
12
|
+
# Helper module that can be included into test cases
|
13
|
+
module EMHelper
|
14
|
+
|
15
|
+
# wraps a given block in a EM event loop and a fiber
|
16
|
+
#
|
17
|
+
# @param [Block] block which is called inside an event loop and a fiber
|
18
|
+
def em(&block)
|
19
|
+
EM.run do
|
20
|
+
Fiber.new do
|
21
|
+
block.call
|
22
|
+
|
23
|
+
EM.stop
|
24
|
+
end.resume
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: junkie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,23 +9,184 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
13
|
-
dependencies:
|
12
|
+
date: 2012-12-02 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: eventmachine
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: em-http-request
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '1.0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '1.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: yajl-ruby
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.1.0
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.1.0
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: nokogiri
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.5'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '1.5'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: hashconfig
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 0.0.1
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 0.0.1
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: serienrenamer
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 0.0.1
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 0.0.1
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: sjunkieex
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.0.1
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: 0.0.1
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: sindex
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: 0.0.1
|
134
|
+
type: :runtime
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: 0.0.1
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: logging
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ~>
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.8.0
|
150
|
+
type: :runtime
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ~>
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: 1.8.0
|
14
158
|
description: TV series management application
|
15
159
|
email:
|
16
160
|
- philipp-boehm@live.de
|
17
|
-
executables:
|
161
|
+
executables:
|
162
|
+
- junkie
|
18
163
|
extensions: []
|
19
164
|
extra_rdoc_files: []
|
20
165
|
files:
|
21
166
|
- .gitignore
|
167
|
+
- .rspec
|
168
|
+
- .travis.yml
|
22
169
|
- Gemfile
|
23
170
|
- LICENSE.txt
|
24
171
|
- README.md
|
25
172
|
- Rakefile
|
173
|
+
- bin/junkie
|
26
174
|
- junkie.gemspec
|
27
175
|
- lib/junkie.rb
|
176
|
+
- lib/junkie/config.rb
|
177
|
+
- lib/junkie/episode.rb
|
178
|
+
- lib/junkie/errors.rb
|
179
|
+
- lib/junkie/helper.rb
|
180
|
+
- lib/junkie/log.rb
|
181
|
+
- lib/junkie/patched/sjunkieex.rb
|
182
|
+
- lib/junkie/pyload/api.rb
|
183
|
+
- lib/junkie/pyload/observer.rb
|
184
|
+
- lib/junkie/reactor.rb
|
28
185
|
- lib/junkie/version.rb
|
186
|
+
- spec/config_spec.rb
|
187
|
+
- spec/environment_spec.rb
|
188
|
+
- spec/pyload_api_spec.rb
|
189
|
+
- spec/spec_helper.rb
|
29
190
|
homepage: https://github.com/pboehm/junkie
|
30
191
|
licenses: []
|
31
192
|
post_install_message:
|
@@ -50,4 +211,9 @@ rubygems_version: 1.8.24
|
|
50
211
|
signing_key:
|
51
212
|
specification_version: 3
|
52
213
|
summary: TV series managament tool that does all the work you have with your series.
|
53
|
-
test_files:
|
214
|
+
test_files:
|
215
|
+
- spec/config_spec.rb
|
216
|
+
- spec/environment_spec.rb
|
217
|
+
- spec/pyload_api_spec.rb
|
218
|
+
- spec/spec_helper.rb
|
219
|
+
has_rdoc:
|