junkie 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +1 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/README.md +1 -1
- data/Rakefile +5 -0
- data/bin/junkie +10 -0
- data/junkie.gemspec +9 -0
- data/lib/junkie/config.rb +75 -0
- data/lib/junkie/episode.rb +30 -0
- data/lib/junkie/errors.rb +13 -0
- data/lib/junkie/helper.rb +13 -0
- data/lib/junkie/log.rb +16 -0
- data/lib/junkie/patched/sjunkieex.rb +79 -0
- data/lib/junkie/pyload/api.rb +161 -0
- data/lib/junkie/pyload/observer.rb +197 -0
- data/lib/junkie/reactor.rb +105 -0
- data/lib/junkie/version.rb +1 -1
- data/lib/junkie.rb +21 -2
- data/spec/config_spec.rb +39 -0
- data/spec/environment_spec.rb +7 -0
- data/spec/pyload_api_spec.rb +118 -0
- data/spec/spec_helper.rb +27 -0
- metadata +171 -5
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
data/Rakefile
CHANGED
data/bin/junkie
ADDED
data/junkie.gemspec
CHANGED
@@ -16,4 +16,13 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
17
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
18
|
gem.require_paths = ["lib"]
|
19
|
+
gem.add_runtime_dependency(%q<eventmachine>, [">= 1.0"])
|
20
|
+
gem.add_runtime_dependency(%q<em-http-request>, [">= 1.0"])
|
21
|
+
gem.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.1.0"])
|
22
|
+
gem.add_runtime_dependency(%q<nokogiri>, [">= 1.5"])
|
23
|
+
gem.add_runtime_dependency(%q<hashconfig>, [">= 0.0.1"])
|
24
|
+
gem.add_runtime_dependency(%q<serienrenamer>, [">= 0.0.1"])
|
25
|
+
gem.add_runtime_dependency(%q<sjunkieex>, [">= 0.0.1"])
|
26
|
+
gem.add_runtime_dependency(%q<sindex>, [">= 0.0.1"])
|
27
|
+
gem.add_runtime_dependency(%q<logging>, ["~> 1.8.0"])
|
19
28
|
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'hashconfig'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module Junkie
|
5
|
+
|
6
|
+
# Module that encapsulates the handling for configuration into a module
|
7
|
+
# which can be included into any Class which includes a constant Hash named
|
8
|
+
# DEFAULT_CONFIG is returned by Config.get_config(self) merged with the
|
9
|
+
# version from the config file
|
10
|
+
#
|
11
|
+
# @example
|
12
|
+
# class Foo
|
13
|
+
# include Config
|
14
|
+
#
|
15
|
+
# DEFAULT_CONFIG = { refresh: 5 }
|
16
|
+
#
|
17
|
+
# def initialize
|
18
|
+
# # the following holds the merged version from DEFAULT_CONFIG
|
19
|
+
# @config = Config.get_config(self)
|
20
|
+
# end
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
module Config
|
24
|
+
@including_classes = []
|
25
|
+
@comlete_config = nil
|
26
|
+
|
27
|
+
# Callback method which is called when a class includes the module
|
28
|
+
#
|
29
|
+
# @param [Class] the class which includes the module
|
30
|
+
def self.included(mod)
|
31
|
+
@including_classes << mod
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
# Get the config hash for the calling module merged with the hash from the
|
36
|
+
# config file
|
37
|
+
#
|
38
|
+
# @param [Class] should be equal to `self`
|
39
|
+
# @return [Hash] merged version of DEFAULT_CONFIG from the calling class
|
40
|
+
def self.get_config(source)
|
41
|
+
|
42
|
+
# builds up the complete config and merges them with serialzed version
|
43
|
+
# at the first call to this method
|
44
|
+
if @comlete_config.nil?
|
45
|
+
default_config = self.collect_default_configs
|
46
|
+
|
47
|
+
FileUtils.mkdir_p File.dirname(Junkie::CONFIG_FILE)
|
48
|
+
@comlete_config =
|
49
|
+
default_config.merge_with_serialized(Junkie::CONFIG_FILE)
|
50
|
+
end
|
51
|
+
|
52
|
+
return @comlete_config[source.class.to_s]
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
# Collects the DEFAULT_CONFIG from all including Classes
|
57
|
+
#
|
58
|
+
# @return [Hash] hash of hashes indexed by class name
|
59
|
+
def self.collect_default_configs
|
60
|
+
config = Hash.new
|
61
|
+
|
62
|
+
@including_classes.each do |klass|
|
63
|
+
if not klass.const_defined? :DEFAULT_CONFIG
|
64
|
+
raise NotImplementedError,
|
65
|
+
"#{klass} has to include a constant DEFAULT_CONFIG as Hash"
|
66
|
+
end
|
67
|
+
|
68
|
+
config[klass.to_s] = klass.const_get :DEFAULT_CONFIG
|
69
|
+
end
|
70
|
+
|
71
|
+
config
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
module Junkie
|
3
|
+
|
4
|
+
require 'sindex'
|
5
|
+
|
6
|
+
class Episode
|
7
|
+
attr_reader :id, :series, :found_at, :link
|
8
|
+
attr_accessor :description, :status
|
9
|
+
|
10
|
+
def initialize(series, link, description=nil)
|
11
|
+
@series = series
|
12
|
+
@link = link
|
13
|
+
@found_at = DateTime.now
|
14
|
+
@description = description
|
15
|
+
@status = :found
|
16
|
+
@id = episode_identifier
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
"%s (%s)" % [ @series, episode_identifier ]
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def episode_identifier
|
26
|
+
Sindex::SeriesIndex.extract_episode_identifier(@description)
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Junkie
|
4
|
+
class InvalidCredentialsError < Exception; end
|
5
|
+
|
6
|
+
class HTTP403Error < Exception; end
|
7
|
+
class HTTP404Error < Exception; end
|
8
|
+
class HTTP500Error < Exception; end
|
9
|
+
|
10
|
+
class InvalidConfigError < Exception; end
|
11
|
+
|
12
|
+
class InvalidStateError < Exception; end
|
13
|
+
end
|
data/lib/junkie/log.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'logging'
|
2
|
+
|
3
|
+
module Junkie
|
4
|
+
|
5
|
+
# Utility module that includes a logging function `log` into the class
|
6
|
+
# that includes this mdoule
|
7
|
+
module Log
|
8
|
+
|
9
|
+
# constructs a new Logger for the class
|
10
|
+
#
|
11
|
+
# @return [Logging.logger] returns a Logger instance for the current class
|
12
|
+
def log
|
13
|
+
@log ||= Logging.logger[self]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'em-http'
|
2
|
+
require 'fiber'
|
3
|
+
require 'junkie/log'
|
4
|
+
require 'sjunkieex'
|
5
|
+
|
6
|
+
module Junkie
|
7
|
+
|
8
|
+
# Monkex-patched version of the Interface from the `sjunkieex` gem
|
9
|
+
class ::Sjunkieex::Interface
|
10
|
+
include Junkie::Log
|
11
|
+
|
12
|
+
|
13
|
+
# Method that searches for new episodes and returns it in a nice structure
|
14
|
+
#
|
15
|
+
# @note should be called inside a Ruby fiber
|
16
|
+
# @return [Hash] series names as keys, data as values
|
17
|
+
def get_links_for_downloads
|
18
|
+
episodes = []
|
19
|
+
look_for_new_episodes.each do |link,series|
|
20
|
+
|
21
|
+
links = parse_series_page(series, link)
|
22
|
+
links.each do |identifier, link_data|
|
23
|
+
|
24
|
+
hd = @options[:hd_enabled]
|
25
|
+
|
26
|
+
# select links, depending on wanted resolution
|
27
|
+
links = []
|
28
|
+
if hd
|
29
|
+
if link_data[:hd_1080p]
|
30
|
+
links = link_data[:hd_1080p]
|
31
|
+
elsif link_data[:hd_720p]
|
32
|
+
links = link_data[:hd_720p]
|
33
|
+
end
|
34
|
+
else
|
35
|
+
(links = link_data[:sd]) if link_data[:sd]
|
36
|
+
end
|
37
|
+
|
38
|
+
if links.empty?
|
39
|
+
log.info("#{series}(#{identifier}) no links in this resolution")
|
40
|
+
next
|
41
|
+
end
|
42
|
+
|
43
|
+
download_links = links.select do |link|
|
44
|
+
link.match(/\/f-\w+\/#{ @options[:hoster_id] }_/)
|
45
|
+
end
|
46
|
+
|
47
|
+
if download_links.empty?
|
48
|
+
puts "there are no links for this hoster"
|
49
|
+
next
|
50
|
+
end
|
51
|
+
|
52
|
+
episode = Junkie::Episode.new(
|
53
|
+
series, download_links.first, link_data[:episodedata])
|
54
|
+
episodes << episode
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
episodes
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def get_page_data(link)
|
64
|
+
f = Fiber.current
|
65
|
+
http = EventMachine::HttpRequest.new(link).get :timeout => 10
|
66
|
+
|
67
|
+
http.callback { f.resume(http) }
|
68
|
+
http.errback { f.resume(http) }
|
69
|
+
|
70
|
+
Fiber.yield
|
71
|
+
|
72
|
+
if http.error
|
73
|
+
raise IOError, http.error
|
74
|
+
end
|
75
|
+
|
76
|
+
http.response
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,161 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'em-http'
|
3
|
+
require 'em-http/middleware/json_response'
|
4
|
+
require 'yaml'
|
5
|
+
require 'json'
|
6
|
+
require 'uri'
|
7
|
+
|
8
|
+
module Junkie
|
9
|
+
module Pyload
|
10
|
+
|
11
|
+
# Class that abstracts the Pyload Api
|
12
|
+
class Api
|
13
|
+
include Config
|
14
|
+
|
15
|
+
attr_reader :config, :cookie
|
16
|
+
|
17
|
+
FILE_STATUS = {
|
18
|
+
0 => :done,
|
19
|
+
2 => :online,
|
20
|
+
3 => :queued,
|
21
|
+
4 => :skipped,
|
22
|
+
5 => :waiting,
|
23
|
+
8 => :failed,
|
24
|
+
10 => :decrypting,
|
25
|
+
12 => :downloading,
|
26
|
+
13 => :extracting,
|
27
|
+
}
|
28
|
+
|
29
|
+
DEFAULT_CONFIG = {
|
30
|
+
protocol: 'http',
|
31
|
+
host: 'localhost',
|
32
|
+
port: 8000,
|
33
|
+
api_user: '',
|
34
|
+
api_password: ''
|
35
|
+
}
|
36
|
+
|
37
|
+
def initialize()
|
38
|
+
@config = Config.get_config(self)
|
39
|
+
|
40
|
+
api_user = @config[:api_user]
|
41
|
+
api_password = @config[:api_password]
|
42
|
+
|
43
|
+
unless api_password && api_user &&
|
44
|
+
api_password.match(/\w+/) && api_user.match(/\w+/)
|
45
|
+
raise InvalidConfigError, "api_user or api_password are not configured"
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
# makes a GET request with the given method name and params
|
51
|
+
#
|
52
|
+
# @param [Symbol] API-method e.g :getQueue, :getPackageData
|
53
|
+
# @param [Hash] params that are a passed as query parameters
|
54
|
+
#
|
55
|
+
# @return [Object] parsed JSON Response
|
56
|
+
def call(method, params={})
|
57
|
+
raise ArgumentError, "`method` is not a Symbol" unless method.is_a? Symbol
|
58
|
+
|
59
|
+
query_parameter = {}
|
60
|
+
params.each do |key, value|
|
61
|
+
if value.is_a? Array
|
62
|
+
query_parameter[key] =
|
63
|
+
URI.encode_www_form_component(JSON.dump(value))
|
64
|
+
else
|
65
|
+
query_parameter[key] =
|
66
|
+
URI.encode_www_form_component('"' + value.to_s + '"')
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
request("/#{method.to_s}", :get, query_parameter)
|
71
|
+
end
|
72
|
+
|
73
|
+
|
74
|
+
# Logs the api user in through sending the credentials to the Api
|
75
|
+
#
|
76
|
+
# @note sets cookie instance variable
|
77
|
+
# @raise [Junkie::InvalidCredentialsError] is raised when the Login fails
|
78
|
+
def login
|
79
|
+
resp = nil
|
80
|
+
|
81
|
+
begin
|
82
|
+
resp = request('/login', :post, {
|
83
|
+
username: @config[:api_user],
|
84
|
+
password: @config[:api_password]
|
85
|
+
}, true)
|
86
|
+
|
87
|
+
rescue Junkie::HTTP403Error => e
|
88
|
+
raise Junkie::InvalidCredentialsError,
|
89
|
+
"the supplied credentials are incorrect"
|
90
|
+
end
|
91
|
+
|
92
|
+
# extract Cookie-ID from Login response
|
93
|
+
if resp.response_header["SET_COOKIE"]
|
94
|
+
header = resp.response_header["SET_COOKIE"]
|
95
|
+
if md = header.match(/^(\S+);/)
|
96
|
+
@cookie = md[1]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# builds up a URL string
|
102
|
+
#
|
103
|
+
# @param [String] path the method specific part of the url
|
104
|
+
# @return [String] complete URI String
|
105
|
+
def build_url(path="")
|
106
|
+
"%s://%s:%d/api%s" %
|
107
|
+
[ @config[:protocol], @config[:host], @config[:port], path ]
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
|
113
|
+
# Utility function for making requests
|
114
|
+
#
|
115
|
+
# @param [String] method specific path e.g /getConfig or /login
|
116
|
+
# @param [Symbol] HTTP method that is issued :get, :post, :delete
|
117
|
+
# @param [Hash] Params that are added to the request, as query parameter
|
118
|
+
# on :get requests and body on :post requests
|
119
|
+
# @param [Boolean] return full response instead of body
|
120
|
+
#
|
121
|
+
# @return [Object] JSON Object that is send from the server or complete
|
122
|
+
# response if the previous argument is set
|
123
|
+
def request(path, method=:get, params={}, complete=false)
|
124
|
+
f = Fiber.current
|
125
|
+
|
126
|
+
request_options = {}
|
127
|
+
if method == :get
|
128
|
+
request_options[:query] = params
|
129
|
+
elsif method == :post
|
130
|
+
request_options[:body] = params
|
131
|
+
end
|
132
|
+
|
133
|
+
if @cookie
|
134
|
+
request_options[:head] = {'cookie' => @cookie}
|
135
|
+
end
|
136
|
+
|
137
|
+
conn = EventMachine::HttpRequest.new(build_url(path))
|
138
|
+
conn.use EventMachine::Middleware::JSONResponse
|
139
|
+
|
140
|
+
http = conn.method(method).call request_options
|
141
|
+
|
142
|
+
http.callback { f.resume(http) }
|
143
|
+
http.errback { f.resume(http) }
|
144
|
+
|
145
|
+
Fiber.yield
|
146
|
+
|
147
|
+
case http.response_header.status
|
148
|
+
when 403
|
149
|
+
raise Junkie::HTTP403Error, "Response has status code 403"
|
150
|
+
when 404
|
151
|
+
raise Junkie::HTTP404Error, "Response has status code 404"
|
152
|
+
when 500
|
153
|
+
raise Junkie::HTTP500Error, "Response has status code 505"
|
154
|
+
end
|
155
|
+
|
156
|
+
return http if complete
|
157
|
+
http.response
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,197 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'junkie/helper'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Junkie
|
6
|
+
module Pyload
|
7
|
+
class Observer
|
8
|
+
include Log, Helper, Config
|
9
|
+
|
10
|
+
DEFAULT_CONFIG = {
|
11
|
+
watchdog_refresh: 10, # interval the watchdog_timer is fired
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize(config={})
|
15
|
+
@config = Config.get_config(self)
|
16
|
+
|
17
|
+
@api = Junkie::Pyload::Api.new
|
18
|
+
|
19
|
+
@ready_for_new_links = true
|
20
|
+
@watchdog = nil
|
21
|
+
@active_downloads = Hash.new
|
22
|
+
end
|
23
|
+
|
24
|
+
# is a method that is called once from inside the reactor and allows us
|
25
|
+
# to register custom actions into the reactor
|
26
|
+
def setup
|
27
|
+
in_fiber {
|
28
|
+
@api.login
|
29
|
+
cleanup
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
# Adds new episode to Pyload which downloads the episode and extracts it
|
34
|
+
#
|
35
|
+
# @param [Junkie::Episode] episode which should be downloaded
|
36
|
+
#
|
37
|
+
# @note should only be called if `is_ready?` returns true
|
38
|
+
def add_episode(episode)
|
39
|
+
raise InvalidStateError, "Observer is not ready" unless is_ready?
|
40
|
+
@ready_for_new_links = false
|
41
|
+
|
42
|
+
episode.status = :downloading
|
43
|
+
|
44
|
+
package = "%s@%s" % [ episode.id, episode.series ]
|
45
|
+
|
46
|
+
pid = @api.call(:addPackage, {name: package, links: [episode.link]})
|
47
|
+
|
48
|
+
@active_downloads[pid] = episode
|
49
|
+
|
50
|
+
log.info("Added #{episode} to Pyload, Pid=#{pid}")
|
51
|
+
|
52
|
+
# start watchdog timer if it does not already run
|
53
|
+
if @watchdog.nil?
|
54
|
+
@watchdog = EM::PeriodicTimer.new(@config[:watchdog_refresh]) do
|
55
|
+
monitor_progress
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# checks if the Observer is ready to add new episodes to Pyload
|
61
|
+
#
|
62
|
+
# @returns [Boolean] true if new links can be added
|
63
|
+
def is_ready?
|
64
|
+
@ready_for_new_links
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
# This method is called from the watchdog timer periodically
|
70
|
+
#
|
71
|
+
# It monitors the download process and reacts depending on the results
|
72
|
+
def monitor_progress
|
73
|
+
log.debug("Watchdog timer has been fired")
|
74
|
+
|
75
|
+
in_fiber {
|
76
|
+
catch(:break) {
|
77
|
+
queue_data = @api.call(:getQueueData)
|
78
|
+
|
79
|
+
if queue_data.empty?
|
80
|
+
log.info("Empty Pyload queue, I cancel the watchdog timer")
|
81
|
+
unschedule_watchdog
|
82
|
+
throw :break
|
83
|
+
end
|
84
|
+
|
85
|
+
# extract package IDs and map them to current downloads
|
86
|
+
update_package_ids(queue_data)
|
87
|
+
|
88
|
+
if has_queue_any_failed_links?(queue_data)
|
89
|
+
log.info("There are failed links in the Queue, will fix this")
|
90
|
+
@api.call(:restartFailed)
|
91
|
+
throw :break
|
92
|
+
end
|
93
|
+
|
94
|
+
# look for complete downloads
|
95
|
+
pids = get_complete_downloads(queue_data)
|
96
|
+
|
97
|
+
if not pids.empty?
|
98
|
+
@api.call(:deletePackages, {pids: pids})
|
99
|
+
log.info("Complete packages are removed from Pyload Queue")
|
100
|
+
|
101
|
+
@ready_for_new_links = true
|
102
|
+
end
|
103
|
+
}
|
104
|
+
}
|
105
|
+
end
|
106
|
+
|
107
|
+
# Searches for failed links in the pyload queue
|
108
|
+
#
|
109
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
110
|
+
#
|
111
|
+
# @return [Boolean] true if there are any failed links, false otherwise
|
112
|
+
def has_queue_any_failed_links?(queue_data)
|
113
|
+
queue_data.each do |package|
|
114
|
+
package['links'].each do |link|
|
115
|
+
status = Pyload::Api::FILE_STATUS[link['status']]
|
116
|
+
return true if status == :failed
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
false
|
121
|
+
end
|
122
|
+
|
123
|
+
# Looks for complete downloads and returns their pids
|
124
|
+
#
|
125
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
126
|
+
#
|
127
|
+
# @return [Array] list of pids there the download is complete
|
128
|
+
def get_complete_downloads(queue_data)
|
129
|
+
pids = []
|
130
|
+
queue_data.each do |package|
|
131
|
+
next unless package['links'].size > 0
|
132
|
+
|
133
|
+
next if package['linksdone'] == 0
|
134
|
+
|
135
|
+
# When extracting is in progress the status of the first link is set
|
136
|
+
# to :extracting
|
137
|
+
extracting = package['links'].select do |link|
|
138
|
+
Pyload::Api::FILE_STATUS[link['status']] == :extracting
|
139
|
+
end
|
140
|
+
next unless extracting.empty?
|
141
|
+
|
142
|
+
sizetotal = package['sizetotal'].to_i
|
143
|
+
sizedone = package['sizedone'].to_i
|
144
|
+
|
145
|
+
if sizetotal > 0 && sizedone > 0
|
146
|
+
|
147
|
+
if sizetotal == sizedone
|
148
|
+
pids << package['pid']
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
pids
|
154
|
+
end
|
155
|
+
|
156
|
+
# extract package IDs and map them to current downloads
|
157
|
+
#
|
158
|
+
# @param [Array] queue_data returned from :getQueueData api method
|
159
|
+
def update_package_ids(queue_data)
|
160
|
+
queue_data.each do |package|
|
161
|
+
pid = package['pid']
|
162
|
+
next if @active_downloads.has_key? pid
|
163
|
+
|
164
|
+
if @active_downloads.has_key? pid-1
|
165
|
+
log.info("Package ID has been changed, I will correct this")
|
166
|
+
@active_downloads[pid] = @active_downloads[pid-1]
|
167
|
+
@active_downloads.delete(pid-1)
|
168
|
+
next
|
169
|
+
end
|
170
|
+
|
171
|
+
raise InvalidStateError,
|
172
|
+
"PackageID #{pid} can't be mapped to active Download"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
# cancels the watchdog timer
|
177
|
+
def unschedule_watchdog
|
178
|
+
if @watchdog_timer
|
179
|
+
@watchdog_timer.cancel
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def cleanup
|
184
|
+
log.debug("Made a cleanup of Pyload's Queue")
|
185
|
+
@api.call(:stopAllDownloads)
|
186
|
+
|
187
|
+
packages = @api.call(:getQueue).map { |e| e['pid'] }
|
188
|
+
|
189
|
+
@api.call(:deletePackages, {pids: packages})
|
190
|
+
log.debug("The following packages have been removed: #{packages}")
|
191
|
+
|
192
|
+
@api.call(:unpauseServer)
|
193
|
+
end
|
194
|
+
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'sindex'
|
3
|
+
require 'sjunkieex'
|
4
|
+
require 'eventmachine'
|
5
|
+
require 'fiber'
|
6
|
+
require 'yaml'
|
7
|
+
require 'junkie'
|
8
|
+
require 'junkie/pyload/api'
|
9
|
+
require 'junkie/pyload/observer'
|
10
|
+
require 'junkie/errors'
|
11
|
+
|
12
|
+
module Junkie
|
13
|
+
|
14
|
+
class Reactor
|
15
|
+
include Log, Helper, Config
|
16
|
+
|
17
|
+
DEFAULT_CONFIG = {
|
18
|
+
:hd_enabled => false,
|
19
|
+
:hoster_id => "ul",
|
20
|
+
:series_index_file => File.join(Dir.home, '.sindex/seriesindex.xml'),
|
21
|
+
:episode_queue_timer_refresh => 5, # in seconds
|
22
|
+
:episode_search_refresh => 15, # in minutes
|
23
|
+
}
|
24
|
+
|
25
|
+
|
26
|
+
def initialize
|
27
|
+
@config = Config.get_config(self)
|
28
|
+
|
29
|
+
@sindex = Sindex::SeriesIndex.new(index_file: @config[:series_index_file])
|
30
|
+
@sjunkieex = Sjunkieex::Interface.new(@sindex, @config)
|
31
|
+
@pyload_observer = Junkie::Pyload::Observer.new()
|
32
|
+
|
33
|
+
@episode_queue = EM::Queue.new
|
34
|
+
@found_episodes = Hash.new
|
35
|
+
build_procs # has to be called here
|
36
|
+
end
|
37
|
+
|
38
|
+
def build_procs
|
39
|
+
|
40
|
+
# Proc that looks for new episodes on Seriesjunkies.org and adds them to
|
41
|
+
# the episode_queue if they are new
|
42
|
+
@look_for_new_episodes = Proc.new {
|
43
|
+
in_fiber {
|
44
|
+
log.info "Looking for new episodes"
|
45
|
+
|
46
|
+
@sjunkieex.get_links_for_downloads.each do |episode|
|
47
|
+
identifier = "%s@%s" % [episode.id, episode.series]
|
48
|
+
|
49
|
+
if not @found_episodes.has_key? identifier
|
50
|
+
log.info("Found new episode '#{episode}'")
|
51
|
+
@episode_queue.push(episode)
|
52
|
+
@found_episodes[identifier] = episode
|
53
|
+
end
|
54
|
+
end
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
# Proc that checks is Pyload-Observer is ready for new episodes and the
|
59
|
+
# episode_queue contains new episodes.
|
60
|
+
#
|
61
|
+
# @note Is called from within the reactor
|
62
|
+
@add_episodes_to_pyload = Proc.new do
|
63
|
+
if @pyload_observer.is_ready?
|
64
|
+
@episode_queue.pop do |episode|
|
65
|
+
log.info("Popped episode '#{episode}' from queue")
|
66
|
+
in_fiber {
|
67
|
+
@pyload_observer.add_episode(episode)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
###########################################################################
|
75
|
+
#################### The Reactor ##########################################
|
76
|
+
###########################################################################
|
77
|
+
def start
|
78
|
+
log.info("Starting Junkie #{Junkie::VERSION}")
|
79
|
+
|
80
|
+
EM.run do
|
81
|
+
|
82
|
+
# do some initialization work
|
83
|
+
@pyload_observer.setup
|
84
|
+
|
85
|
+
# Look for new episodes and add them to the Queue if they haven't
|
86
|
+
# been found yet
|
87
|
+
EM.next_tick do
|
88
|
+
@look_for_new_episodes.call
|
89
|
+
|
90
|
+
EM.add_periodic_timer(@config[:episode_search_refresh] * 60) do
|
91
|
+
@look_for_new_episodes.call
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# Add found episodes into Pyload if there are any episodes and pyload
|
96
|
+
# is ready
|
97
|
+
EM.add_periodic_timer(
|
98
|
+
@config[:episode_queue_timer_refresh], @add_episodes_to_pyload)
|
99
|
+
|
100
|
+
# for determining blocking operations
|
101
|
+
# EM.add_periodic_timer(1) { puts Time.now.to_i }
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/junkie/version.rb
CHANGED
data/lib/junkie.rb
CHANGED
@@ -1,5 +1,24 @@
|
|
1
|
-
require
|
1
|
+
require 'junkie/version'
|
2
|
+
require 'junkie/config'
|
3
|
+
require 'junkie/episode'
|
4
|
+
require 'junkie/log'
|
5
|
+
require 'junkie/reactor'
|
6
|
+
require 'junkie/errors'
|
7
|
+
require 'junkie/helper'
|
8
|
+
require 'junkie/patched/sjunkieex'
|
9
|
+
require 'junkie/pyload/api'
|
10
|
+
require 'junkie/pyload/observer'
|
11
|
+
require 'logging'
|
12
|
+
|
13
|
+
Logging.logger.root.appenders = Logging.appenders.stdout
|
14
|
+
Logging.logger.root.level = :debug
|
2
15
|
|
3
16
|
module Junkie
|
4
|
-
|
17
|
+
|
18
|
+
CONFIG_FILE = File.join(Dir.home, '.junkie', 'config.yml')
|
19
|
+
|
20
|
+
def start_reactor
|
21
|
+
r = Reactor.new
|
22
|
+
r.start
|
23
|
+
end
|
5
24
|
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
class ConfigTest
|
5
|
+
include Junkie::Config
|
6
|
+
|
7
|
+
attr_reader :config
|
8
|
+
|
9
|
+
DEFAULT_CONFIG = {
|
10
|
+
refresh: 5,
|
11
|
+
numbers: [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize()
|
15
|
+
@config = Junkie::Config.get_config(self)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe ConfigTest do
|
20
|
+
|
21
|
+
before do
|
22
|
+
@file = '/tmp/testtest.yml'
|
23
|
+
stub_const("Junkie::CONFIG_FILE", @file)
|
24
|
+
end
|
25
|
+
|
26
|
+
after(:each) do
|
27
|
+
FileUtils.rm(@file) if File.file? @file
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should stub out the Config File" do
|
31
|
+
expect(Junkie::CONFIG_FILE).to eq('/tmp/testtest.yml')
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should return the default if there are no changes made" do
|
35
|
+
test = ConfigTest.new
|
36
|
+
expect(test.config).to eq ConfigTest::DEFAULT_CONFIG
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
# WebMock.disable_net_connect!
|
5
|
+
|
6
|
+
describe Junkie::Pyload::Api do
|
7
|
+
include EMHelper
|
8
|
+
|
9
|
+
# remove the Config module behaviour
|
10
|
+
before(:each) do
|
11
|
+
Junkie::Config.stub!(:get_config).and_return(
|
12
|
+
Junkie::Pyload::Api::DEFAULT_CONFIG.merge(
|
13
|
+
{api_user: 'user', api_password: 'pass'}))
|
14
|
+
end
|
15
|
+
|
16
|
+
context "standard configuration" do
|
17
|
+
|
18
|
+
before(:each) do
|
19
|
+
@api = Junkie::Pyload::Api.new()
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should have the right standard url" do
|
23
|
+
expect(@api.build_url).to eq "http://localhost:8000/api"
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
context "#login" do
|
29
|
+
|
30
|
+
context "valid credentials" do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@api = Junkie::Pyload::Api.new()
|
34
|
+
@stub = stub_request(:post, "http://localhost:8000/api/login").
|
35
|
+
with(:body => "username=user&password=pass").
|
36
|
+
to_return(:status => 200, :body => "",
|
37
|
+
:headers => {SET_COOKIE: "api.id=123456; valid until it is removed"})
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should use the right url for API login" do
|
41
|
+
em {
|
42
|
+
@api.login
|
43
|
+
@stub.should have_been_requested
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should extract the Cookie from the login response" do
|
48
|
+
em {
|
49
|
+
@api.login
|
50
|
+
expect(@api.cookie).to eq "api.id=123456"
|
51
|
+
}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context "invalid credentials" do
|
56
|
+
|
57
|
+
before(:each) do
|
58
|
+
@api = Junkie::Pyload::Api.new()
|
59
|
+
@api.config[:api_password] = "invalid"
|
60
|
+
@stub = stub_request(:post, "http://localhost:8000/api/login").
|
61
|
+
with(:body => "username=user&password=invalid").
|
62
|
+
to_return(:status => 403, :body => "", :headers => {})
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should raise an error if login does not work" do
|
66
|
+
em {
|
67
|
+
expect {
|
68
|
+
@api.login
|
69
|
+
}.to raise_error(Junkie::InvalidCredentialsError)
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
context "#call" do
|
77
|
+
|
78
|
+
before(:each) do
|
79
|
+
@api = Junkie::Pyload::Api.new()
|
80
|
+
@api.stub(:cookie).and_return("api.id=123456")
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should call the right url for the method name" do
|
84
|
+
em {
|
85
|
+
stub = stub_request(:get, "http://localhost:8000/api/getQueue")
|
86
|
+
@api.call(:getQueue)
|
87
|
+
stub.should have_been_requested
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should pass parameters as JSON to the Request" do
|
92
|
+
em {
|
93
|
+
stub_request(:get, "http://localhost:8000/api/addPackage?links=%255B%2522http%253A%252F%252Ftest.test.de%2522%255D&name=%2522testpkg%2522").
|
94
|
+
to_return(:status => 200, :body => "76",
|
95
|
+
:headers => {"Content-Type" => "application/json"})
|
96
|
+
|
97
|
+
res = @api.call(:addPackage, { name: "testpkg", links: [ "http://test.test.de" ]})
|
98
|
+
res.should eq 76
|
99
|
+
}
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should build ruby datatypes from the JSON" do
|
103
|
+
em {
|
104
|
+
stub_request(:get, "http://localhost:8000/api/statusServer").
|
105
|
+
to_return(:status => 200, :body => '{ "pause": false, "total": 141 }',
|
106
|
+
:headers => {"Content-Type" => "application/json"})
|
107
|
+
|
108
|
+
res = @api.call(:statusServer)
|
109
|
+
hash = { "pause" => false, "total" => 141 }
|
110
|
+
res.should eq hash
|
111
|
+
}
|
112
|
+
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
end
|
118
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../lib/junkie'
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'rspec'
|
5
|
+
require 'eventmachine'
|
6
|
+
require 'fiber'
|
7
|
+
require 'webmock/rspec'
|
8
|
+
|
9
|
+
RSpec.configure do |config|
|
10
|
+
end
|
11
|
+
|
12
|
+
# Helper module that can be included into test cases
|
13
|
+
module EMHelper
|
14
|
+
|
15
|
+
# wraps a given block in a EM event loop and a fiber
|
16
|
+
#
|
17
|
+
# @param [Block] block which is called inside an event loop and a fiber
|
18
|
+
def em(&block)
|
19
|
+
EM.run do
|
20
|
+
Fiber.new do
|
21
|
+
block.call
|
22
|
+
|
23
|
+
EM.stop
|
24
|
+
end.resume
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: junkie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,23 +9,184 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
13
|
-
dependencies:
|
12
|
+
date: 2012-12-02 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: eventmachine
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: em-http-request
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '1.0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '1.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: yajl-ruby
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.1.0
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.1.0
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: nokogiri
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.5'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '1.5'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: hashconfig
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 0.0.1
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 0.0.1
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: serienrenamer
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 0.0.1
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 0.0.1
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: sjunkieex
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.0.1
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: 0.0.1
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: sindex
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: 0.0.1
|
134
|
+
type: :runtime
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: 0.0.1
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: logging
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ~>
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.8.0
|
150
|
+
type: :runtime
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ~>
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: 1.8.0
|
14
158
|
description: TV series management application
|
15
159
|
email:
|
16
160
|
- philipp-boehm@live.de
|
17
|
-
executables:
|
161
|
+
executables:
|
162
|
+
- junkie
|
18
163
|
extensions: []
|
19
164
|
extra_rdoc_files: []
|
20
165
|
files:
|
21
166
|
- .gitignore
|
167
|
+
- .rspec
|
168
|
+
- .travis.yml
|
22
169
|
- Gemfile
|
23
170
|
- LICENSE.txt
|
24
171
|
- README.md
|
25
172
|
- Rakefile
|
173
|
+
- bin/junkie
|
26
174
|
- junkie.gemspec
|
27
175
|
- lib/junkie.rb
|
176
|
+
- lib/junkie/config.rb
|
177
|
+
- lib/junkie/episode.rb
|
178
|
+
- lib/junkie/errors.rb
|
179
|
+
- lib/junkie/helper.rb
|
180
|
+
- lib/junkie/log.rb
|
181
|
+
- lib/junkie/patched/sjunkieex.rb
|
182
|
+
- lib/junkie/pyload/api.rb
|
183
|
+
- lib/junkie/pyload/observer.rb
|
184
|
+
- lib/junkie/reactor.rb
|
28
185
|
- lib/junkie/version.rb
|
186
|
+
- spec/config_spec.rb
|
187
|
+
- spec/environment_spec.rb
|
188
|
+
- spec/pyload_api_spec.rb
|
189
|
+
- spec/spec_helper.rb
|
29
190
|
homepage: https://github.com/pboehm/junkie
|
30
191
|
licenses: []
|
31
192
|
post_install_message:
|
@@ -50,4 +211,9 @@ rubygems_version: 1.8.24
|
|
50
211
|
signing_key:
|
51
212
|
specification_version: 3
|
52
213
|
summary: TV series managament tool that does all the work you have with your series.
|
53
|
-
test_files:
|
214
|
+
test_files:
|
215
|
+
- spec/config_spec.rb
|
216
|
+
- spec/environment_spec.rb
|
217
|
+
- spec/pyload_api_spec.rb
|
218
|
+
- spec/spec_helper.rb
|
219
|
+
has_rdoc:
|