storyboard 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/.gitmodules +3 -0
- data/.rvmrc +1 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +55 -0
- data/README.md +40 -0
- data/TODO +6 -0
- data/bin/storyboard +82 -0
- data/bin/storyboard-ffprobe +0 -0
- data/lib/.DS_Store +0 -0
- data/lib/storyboard/generators/pdf.rb +46 -0
- data/lib/storyboard/generators/sub.rb +32 -0
- data/lib/storyboard/subtitles.rb +96 -0
- data/lib/storyboard/thread-util.rb +308 -0
- data/lib/storyboard/time.rb +34 -0
- data/lib/storyboard/version.rb +3 -0
- data/lib/storyboard.rb +119 -0
- data/storyboard.gemspec +56 -0
- data/vendor/suby/.gitignore +3 -0
- data/vendor/suby/LICENSE +19 -0
- data/vendor/suby/README.md +27 -0
- data/vendor/suby/bin/suby +30 -0
- data/vendor/suby/lib/suby/downloader/addic7ed.rb +65 -0
- data/vendor/suby/lib/suby/downloader/opensubtitles.rb +83 -0
- data/vendor/suby/lib/suby/downloader/tvsubtitles.rb +90 -0
- data/vendor/suby/lib/suby/downloader.rb +177 -0
- data/vendor/suby/lib/suby/filename_parser.rb +103 -0
- data/vendor/suby/lib/suby/interface.rb +17 -0
- data/vendor/suby/lib/suby/movie_hasher.rb +31 -0
- data/vendor/suby/lib/suby.rb +89 -0
- data/vendor/suby/spec/fixtures/.gitkeep +0 -0
- data/vendor/suby/spec/mock_http.rb +22 -0
- data/vendor/suby/spec/spec_helper.rb +3 -0
- data/vendor/suby/spec/suby/downloader/addict7ed_spec.rb +28 -0
- data/vendor/suby/spec/suby/downloader/opensubtitles_spec.rb +33 -0
- data/vendor/suby/spec/suby/downloader/tvsubtitles_spec.rb +50 -0
- data/vendor/suby/spec/suby/downloader_spec.rb +11 -0
- data/vendor/suby/spec/suby/filename_parser_spec.rb +66 -0
- data/vendor/suby/spec/suby_spec.rb +27 -0
- data/vendor/suby/suby.gemspec +20 -0
- metadata +232 -0
data/.gitignore
ADDED
data/.gitmodules
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm --create use 1.9.3@storyboard
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
GIT
|
2
|
+
remote: http://github.com/markolson/suby
|
3
|
+
revision: f44e9821f0f6ff07d2729a3a0b3abc386861b9e9
|
4
|
+
specs:
|
5
|
+
suby (0.4.0)
|
6
|
+
mime-types (>= 1.19)
|
7
|
+
nokogiri
|
8
|
+
path (>= 1.3.0)
|
9
|
+
rubyzip
|
10
|
+
term-ansicolor
|
11
|
+
|
12
|
+
PATH
|
13
|
+
remote: .
|
14
|
+
specs:
|
15
|
+
Storyboard (0.1.1)
|
16
|
+
nokogiri
|
17
|
+
prawn
|
18
|
+
rmagick
|
19
|
+
ruby-progressbar
|
20
|
+
|
21
|
+
GEM
|
22
|
+
remote: http://rubygems.org/
|
23
|
+
specs:
|
24
|
+
Ascii85 (1.0.2)
|
25
|
+
afm (0.2.0)
|
26
|
+
hashery (2.1.0)
|
27
|
+
mime-types (1.19)
|
28
|
+
nokogiri (1.5.6)
|
29
|
+
path (1.3.1)
|
30
|
+
pdf-reader (1.3.0)
|
31
|
+
Ascii85 (~> 1.0.0)
|
32
|
+
afm (~> 0.2.0)
|
33
|
+
hashery (~> 2.0)
|
34
|
+
ruby-rc4
|
35
|
+
ttfunk
|
36
|
+
prawn (0.12.0)
|
37
|
+
pdf-reader (>= 0.9.0)
|
38
|
+
ttfunk (~> 1.0.2)
|
39
|
+
rmagick (2.13.1)
|
40
|
+
ruby-progressbar (1.0.2)
|
41
|
+
ruby-rc4 (0.1.5)
|
42
|
+
rubyzip (0.9.9)
|
43
|
+
term-ansicolor (1.0.7)
|
44
|
+
ttfunk (1.0.3)
|
45
|
+
|
46
|
+
PLATFORMS
|
47
|
+
ruby
|
48
|
+
|
49
|
+
DEPENDENCIES
|
50
|
+
Storyboard!
|
51
|
+
nokogiri
|
52
|
+
prawn
|
53
|
+
rmagick
|
54
|
+
ruby-progressbar
|
55
|
+
suby!
|
data/README.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# Storyboard
|
2
|
+
|
3
|
+
Read the TV and Movies you don't have time to watch.
|
4
|
+
|
5
|
+
## Storyboard
|
6
|
+
|
7
|
+
Storyboard is _very much_ a work in progress, and only works (most of) some of the time. Using it is simple:
|
8
|
+
|
9
|
+
storyboard /path/to/video-file.mkv
|
10
|
+
|
11
|
+
Storyboard will try to generate a file at `/path/to/video-file.pdf` containing the final product. ePub and Mobi support will come later.
|
12
|
+
|
13
|
+
You can see available commands by running the program without any options
|
14
|
+
|
15
|
+
Usage: storyboard [options] videofile [output_directory]
|
16
|
+
-v, --[no-]verbose Run verbosely
|
17
|
+
--[no-]scenes Detect scene changes. This increases the time it takes to generate a file.
|
18
|
+
-ct FLOAT Scene detection threshold. 0.2 is too low, 0.8 is too high. Play with it!
|
19
|
+
-s, --subs FILE SRT subtitle file to use. Will skip extracting/downloading one.
|
20
|
+
--make x,y,z Filetypes to output
|
21
|
+
(pdf, mobi, epub)
|
22
|
+
-h, --help Show this message
|
23
|
+
|
24
|
+
## Requirements
|
25
|
+
|
26
|
+
Storyboard requires a recent version of *ffmpeg*. This gem includes a build of *ffprobe* that will only work on OS X 1.8, 64 bit. It's probably best not to use this on any different system for now. It's also best to run it on something with at least 8 cores.
|
27
|
+
|
28
|
+
## Known Issues
|
29
|
+
|
30
|
+
* If there is a scene change followed by dialog in the next frame or two, the dialog may not be shown.
|
31
|
+
* Subtitles are always downloaded, never extracted from video files. Because the subtitles are searched for based on the filename it's best that you have then named in a standard format, e.g., `The X-Files - 1x21 - Tooms.avi`.
|
32
|
+
* Sometimes the wrong subtitle file will be returned from the site. In those cases, download it manually and use the `-s` option to pass in the path to an SRT formatted subtitle file.
|
33
|
+
* Some subtitles are encoded in UTF-16, and I haven't care quite enough yet to get them to work.
|
34
|
+
* The subtitles are uuuuuugly.
|
35
|
+
* Hardcoding 8 for the number of threads is a bad idea
|
36
|
+
* Almost definitely some path-escaping issues, so avoid files with apostrophes and slashes
|
37
|
+
|
38
|
+
## Help
|
39
|
+
|
40
|
+
For now, best to email me theothermarkolson@gmail.com
|
data/TODO
ADDED
data/bin/storyboard
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'shellwords'
|
5
|
+
require 'optparse'
|
6
|
+
require 'logger'
|
7
|
+
require 'open3'
|
8
|
+
require 'bundler'
|
9
|
+
ENV['BUNDLE_GEMFILE'] = File.join(File.dirname(__FILE__), '../Gemfile')
|
10
|
+
Bundler.require(:default)
|
11
|
+
|
12
|
+
require 'nokogiri'
|
13
|
+
require 'suby'
|
14
|
+
require 'rmagick'
|
15
|
+
require 'prawn'
|
16
|
+
require 'ruby-progressbar'
|
17
|
+
|
18
|
+
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
|
19
|
+
require 'storyboard'
|
20
|
+
|
21
|
+
options = {:scenes => true, :verbose => true, :types => ['pdf'], :scene_threshold => 0.4}
|
22
|
+
|
23
|
+
# think about them
|
24
|
+
options[:consolidate_frame_threshold] = 0.4
|
25
|
+
options[:max_width] = 600
|
26
|
+
|
27
|
+
LOG = Logger.new(STDOUT)
|
28
|
+
LOG.level = Logger::INFO
|
29
|
+
|
30
|
+
opts = OptionParser.new
|
31
|
+
opts.banner = "Usage: storyboard [options] videofile [output_directory]"
|
32
|
+
|
33
|
+
opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
|
34
|
+
options[:version] = v
|
35
|
+
LOG.level = Logger::DEBUG if v
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("-c", "--[no-]scenes", "Detect scene changes. This increases the time it takes to generate a file.") do |v|
|
39
|
+
options[:scenes] = v
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on("-ct", Float, "Scene detection threshold. 0.2 is too low, 0.8 is too high. Play with it!") do |f|
|
43
|
+
options[:scene_threshold] = f
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.on("-s", "--subs FILE", "SRT subtitle file to use. Will skip extracting/downloading one.") do |s|
|
47
|
+
options[:subs] = s
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on("--make x,y,z", Array, "Filetypes to output", '(pdf, mobi, epub)') do |types|
|
51
|
+
options[:types] = types
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
55
|
+
puts opts
|
56
|
+
exit
|
57
|
+
end
|
58
|
+
|
59
|
+
begin opts.parse! ARGV
|
60
|
+
rescue OptionParser::InvalidOption, OptionParser::InvalidArgument => e
|
61
|
+
puts e
|
62
|
+
puts opts
|
63
|
+
exit 1
|
64
|
+
end
|
65
|
+
|
66
|
+
if ARGV.size < 1
|
67
|
+
puts "videofile required"
|
68
|
+
puts opts.to_s
|
69
|
+
exit 1
|
70
|
+
end
|
71
|
+
|
72
|
+
options[:file] = File.realdirpath(ARGV.shift)
|
73
|
+
|
74
|
+
output_dir = ARGV.shift || Dir.pwd
|
75
|
+
options[:basename] = File.basename(options[:file], ".*")
|
76
|
+
|
77
|
+
options[:write_to] = output_dir #File.join(output_dir,options[:basename])
|
78
|
+
options[:work_dir] = Dir.mktmpdir
|
79
|
+
Dir.mkdir(options[:write_to]) unless File.directory?(options[:write_to])
|
80
|
+
LOG.debug(options)
|
81
|
+
|
82
|
+
Storyboard.new(options)
|
Binary file
|
data/lib/.DS_Store
ADDED
Binary file
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'prawn'
|
2
|
+
|
3
|
+
require 'rmagick'
|
4
|
+
include Magick
|
5
|
+
|
6
|
+
class Storyboard
|
7
|
+
class PDFRenderer < Storyboard::Renderer
|
8
|
+
|
9
|
+
attr_accessor :pdf, :storyboard, :dimensions
|
10
|
+
def initialize(parent)
|
11
|
+
@dimensions = []
|
12
|
+
@storyboard = parent
|
13
|
+
end
|
14
|
+
|
15
|
+
def set_dimensions(w,h)
|
16
|
+
@dimensions = [w,h]
|
17
|
+
@pdf = Prawn::Document.new(:page_size => [w, h], :margin => 0)
|
18
|
+
end
|
19
|
+
|
20
|
+
def write
|
21
|
+
@pdf.render_file "#{@storyboard.options[:write_to]}/#{@storyboard.options[:basename]}.pdf"
|
22
|
+
LOG.info("Wrote #{@storyboard.options[:write_to]}/#{@storyboard.options[:basename]}.pdf")
|
23
|
+
end
|
24
|
+
|
25
|
+
def render_frame(frame_name, subtitle = nil)
|
26
|
+
image_output = File.join(@storyboard.options[:save_directory], "sub-#{File.basename(frame_name)}")
|
27
|
+
img = ImageList.new(frame_name)
|
28
|
+
|
29
|
+
if(@dimensions.empty?)
|
30
|
+
resize_height = (img.rows * (@storyboard.options[:max_width].to_f/img.columns)).ceil
|
31
|
+
set_dimensions(storyboard.options[:max_width], resize_height)
|
32
|
+
end
|
33
|
+
|
34
|
+
img.resize_to_fit!(@dimensions[0], @dimensions[1])
|
35
|
+
|
36
|
+
self.add_subtitle(img, subtitle) if subtitle
|
37
|
+
|
38
|
+
img.format = 'jpeg'
|
39
|
+
img.write(image_output) { self.quality = 60 }
|
40
|
+
img.destroy!
|
41
|
+
|
42
|
+
@pdf.image image_output, :width => @dimensions[0], :height => @dimensions[1]
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class Storyboard
|
2
|
+
class Renderer
|
3
|
+
def add_subtitle(img, subtitle)
|
4
|
+
text = subtitle.lines.join("\n")
|
5
|
+
txt = nil
|
6
|
+
txtwidth = img.columns + 1
|
7
|
+
txtsize = 29
|
8
|
+
while(txtwidth > (img.columns * 0.8))
|
9
|
+
txtsize -= 1
|
10
|
+
txt = Draw.new
|
11
|
+
txt.pointsize = txtsize
|
12
|
+
o = txt.get_multiline_type_metrics(img, text)
|
13
|
+
txtwidth = o.width
|
14
|
+
end
|
15
|
+
|
16
|
+
txt.gravity = Magick::SouthGravity
|
17
|
+
txt.stroke_width = 1
|
18
|
+
txt.stroke = 'transparent'
|
19
|
+
txt.font_weight = Magick::BoldWeight
|
20
|
+
|
21
|
+
img.annotate(txt, 0,0,-2,-2, text) {
|
22
|
+
txt.fill = '#333333'
|
23
|
+
}
|
24
|
+
|
25
|
+
img.annotate(txt, 0,0,0,0, text){
|
26
|
+
txt.fill = "#ffffff"
|
27
|
+
txt.stroke = "#000000"
|
28
|
+
}
|
29
|
+
txt = nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
class Storyboard
|
2
|
+
def get_subtitles
|
3
|
+
if false == "the file has subtitles embedded"
|
4
|
+
|
5
|
+
else
|
6
|
+
LOG.debug("No subtitles embeded. Using suby.")
|
7
|
+
# suby includes a giant util library the guy also wrote
|
8
|
+
# that it uses to call file.basename instead of File.basename(file),
|
9
|
+
#but "file" has to be a "Path", so, whatever.
|
10
|
+
suby_file = Path(options[:file])
|
11
|
+
downloader = Suby::Downloader::OpenSubtitles.new(suby_file, 'en')
|
12
|
+
LOG.debug("Found #{downloader.download_url}")
|
13
|
+
downloader.extract(downloader.download_url)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
class SRT
|
19
|
+
Page = Struct.new(:index, :start_time, :end_time, :lines)
|
20
|
+
|
21
|
+
TIME_REGEX = /\d{2}:\d{2}:\d{2}[,\.]\d{1,4}/
|
22
|
+
attr_accessor :text, :pages, :options
|
23
|
+
|
24
|
+
def initialize(contents, parent_options)
|
25
|
+
@options = parent_options
|
26
|
+
@text = contents
|
27
|
+
@pages = []
|
28
|
+
parse
|
29
|
+
clean_promos
|
30
|
+
LOG.info("Parsed subtitle file. #{count} entries found.")
|
31
|
+
end
|
32
|
+
|
33
|
+
#There are some horrid files, so I want to be able to have more than just a single regex
|
34
|
+
#to parse the srt file. Eventually, handling these errors will be a thing to do.
|
35
|
+
def parse
|
36
|
+
phase = :line_no
|
37
|
+
page = nil
|
38
|
+
@text.each_line {|l|
|
39
|
+
l = l.strip
|
40
|
+
# Some files have BOM markers. Why? Why would you add a BOM marker.
|
41
|
+
l.gsub!("\xEF\xBB\xBF".force_encoding("UTF-8"), '') if page.nil?
|
42
|
+
case phase
|
43
|
+
when :line_no
|
44
|
+
if l =~ /^\d+$/
|
45
|
+
page = Page.new(@pages.count + 1, nil, nil, [])
|
46
|
+
phase = :time
|
47
|
+
elsif !l.empty?
|
48
|
+
raise "Bad SRT File: Should have a block number but got '#{l}' [#{l.bytes.to_a.join(',')}]"
|
49
|
+
end
|
50
|
+
when :time
|
51
|
+
if l =~ /^(#{TIME_REGEX}) --> (#{TIME_REGEX})$/
|
52
|
+
page[:start_time] = STRTime.parse($1)
|
53
|
+
page[:end_time] = STRTime.parse($2)
|
54
|
+
phase = :text
|
55
|
+
else
|
56
|
+
raise "Bad SRT File: Should have time range but got '#{l}'"
|
57
|
+
end
|
58
|
+
when :text
|
59
|
+
if l.empty?
|
60
|
+
phase = :line_no
|
61
|
+
@pages << page
|
62
|
+
else
|
63
|
+
page[:lines] << l.gsub(%r{</?[^>]+?>}, '')
|
64
|
+
end
|
65
|
+
end
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
# Strip out obnoxious "CREATED BY L33T DUD3" or "DOWNLOADED FROM ____" text
|
70
|
+
def clean_promos
|
71
|
+
@pages.delete_if {|page|
|
72
|
+
!page[:lines].grep(/Subtitles downloaded/).empty? ||
|
73
|
+
!page[:lines].grep(/addic7ed/).empty? ||
|
74
|
+
!page[:lines].grep(/OpenSubtitles/).empty? ||
|
75
|
+
!page[:lines].grep(/sync, corrected by/).empty? ||
|
76
|
+
false
|
77
|
+
}
|
78
|
+
end
|
79
|
+
|
80
|
+
def save
|
81
|
+
File.open(File.join(options[:work_dir], options[:basename] + '.srt'), 'w') {|f|
|
82
|
+
f.write(self.to_s)
|
83
|
+
}
|
84
|
+
self
|
85
|
+
end
|
86
|
+
|
87
|
+
def to_s
|
88
|
+
text
|
89
|
+
end
|
90
|
+
|
91
|
+
def count
|
92
|
+
@pages.count
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,308 @@
|
|
1
|
+
#--
|
2
|
+
# DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
|
3
|
+
# Version 2, December 2004
|
4
|
+
#
|
5
|
+
# DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
|
6
|
+
# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
7
|
+
#
|
8
|
+
# 0. You just DO WHAT THE FUCK YOU WANT TO.
|
9
|
+
#++
|
10
|
+
|
11
|
+
require 'thread'
|
12
|
+
class Thread::Pool
|
13
|
+
class Task
|
14
|
+
Timeout = Class.new(Exception)
|
15
|
+
Asked = Class.new(Exception)
|
16
|
+
|
17
|
+
attr_reader :pool, :timeout, :exception, :thread, :started_at
|
18
|
+
|
19
|
+
def initialize (pool, *args, &block)
|
20
|
+
@pool = pool
|
21
|
+
@arguments = args
|
22
|
+
@block = block
|
23
|
+
end
|
24
|
+
|
25
|
+
def running?; @running; end
|
26
|
+
def finished?; @finished; end
|
27
|
+
def timeout?; @timedout; end
|
28
|
+
def terminated?; @terminated; end
|
29
|
+
|
30
|
+
def execute (thread)
|
31
|
+
return if terminated? || running? || finished?
|
32
|
+
|
33
|
+
@thread = thread
|
34
|
+
@running = true
|
35
|
+
@started_at = Time.now
|
36
|
+
|
37
|
+
pool.wake_up_timeout
|
38
|
+
|
39
|
+
begin
|
40
|
+
@block.call(*@arguments)
|
41
|
+
rescue Exception => reason
|
42
|
+
if reason.is_a? Timeout
|
43
|
+
@timedout = true
|
44
|
+
elsif reason.is_a? Asked
|
45
|
+
return
|
46
|
+
else
|
47
|
+
@exception = reason
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
@running = false
|
52
|
+
@finished = true
|
53
|
+
@thread = nil
|
54
|
+
end
|
55
|
+
|
56
|
+
def terminate! (exception = Asked)
|
57
|
+
return if terminated? || finished? || timeout?
|
58
|
+
|
59
|
+
@terminated = true
|
60
|
+
|
61
|
+
return unless running?
|
62
|
+
|
63
|
+
@thread.raise exception
|
64
|
+
end
|
65
|
+
|
66
|
+
def timeout!
|
67
|
+
terminate! Timeout
|
68
|
+
end
|
69
|
+
|
70
|
+
def timeout_after (time)
|
71
|
+
@timeout = time
|
72
|
+
|
73
|
+
pool.timeout_for self, time
|
74
|
+
|
75
|
+
self
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
attr_reader :min, :max, :spawned
|
80
|
+
|
81
|
+
def initialize (min, max = nil, &block)
|
82
|
+
@min = min
|
83
|
+
@max = max || min
|
84
|
+
@block = block
|
85
|
+
|
86
|
+
@cond = ConditionVariable.new
|
87
|
+
@mutex = Mutex.new
|
88
|
+
|
89
|
+
@todo = []
|
90
|
+
@workers = []
|
91
|
+
@timeouts = {}
|
92
|
+
|
93
|
+
@spawned = 0
|
94
|
+
@waiting = 0
|
95
|
+
@shutdown = false
|
96
|
+
@trim_requests = 0
|
97
|
+
@auto_trim = false
|
98
|
+
|
99
|
+
@mutex.synchronize {
|
100
|
+
min.times {
|
101
|
+
spawn_thread
|
102
|
+
}
|
103
|
+
}
|
104
|
+
end
|
105
|
+
|
106
|
+
def shutdown?; !!@shutdown; end
|
107
|
+
|
108
|
+
def auto_trim?; @auto_trim; end
|
109
|
+
def auto_trim!; @auto_trim = true; end
|
110
|
+
def no_auto_trim!; @auto_trim = false; end
|
111
|
+
|
112
|
+
def resize (min, max = nil)
|
113
|
+
@min = min
|
114
|
+
@max = max || min
|
115
|
+
|
116
|
+
trim!
|
117
|
+
end
|
118
|
+
|
119
|
+
def backlog
|
120
|
+
@mutex.synchronize {
|
121
|
+
@todo.length
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
125
|
+
def process (*args, &block)
|
126
|
+
unless block || @block
|
127
|
+
raise ArgumentError, 'you must pass a block'
|
128
|
+
end
|
129
|
+
|
130
|
+
task = Task.new(self, *args, &(block || @block))
|
131
|
+
|
132
|
+
@mutex.synchronize {
|
133
|
+
raise 'unable to add work while shutting down' if shutdown?
|
134
|
+
|
135
|
+
@todo << task
|
136
|
+
|
137
|
+
if @waiting == 0 && @spawned < @max
|
138
|
+
spawn_thread
|
139
|
+
end
|
140
|
+
|
141
|
+
@cond.signal
|
142
|
+
}
|
143
|
+
|
144
|
+
task
|
145
|
+
end
|
146
|
+
|
147
|
+
alias << process
|
148
|
+
|
149
|
+
def trim (force = false)
|
150
|
+
@mutex.synchronize {
|
151
|
+
if (force || @waiting > 0) && @spawned - @trim_requests > @min
|
152
|
+
@trim_requests -= 1
|
153
|
+
@cond.signal
|
154
|
+
end
|
155
|
+
}
|
156
|
+
|
157
|
+
self
|
158
|
+
end
|
159
|
+
|
160
|
+
def trim!
|
161
|
+
trim true
|
162
|
+
end
|
163
|
+
|
164
|
+
def shutdown!
|
165
|
+
@mutex.synchronize {
|
166
|
+
@shutdown = :now
|
167
|
+
@cond.broadcast
|
168
|
+
}
|
169
|
+
|
170
|
+
wake_up_timeout
|
171
|
+
|
172
|
+
self
|
173
|
+
end
|
174
|
+
|
175
|
+
def shutdown
|
176
|
+
@mutex.synchronize {
|
177
|
+
@shutdown = :nicely
|
178
|
+
@cond.broadcast
|
179
|
+
}
|
180
|
+
|
181
|
+
join
|
182
|
+
|
183
|
+
if @timeout
|
184
|
+
@shutdown = :now
|
185
|
+
|
186
|
+
wake_up_timeout
|
187
|
+
|
188
|
+
@timeout.join
|
189
|
+
end
|
190
|
+
|
191
|
+
self
|
192
|
+
end
|
193
|
+
|
194
|
+
def join
|
195
|
+
@workers.first.join until @workers.empty?
|
196
|
+
|
197
|
+
self
|
198
|
+
end
|
199
|
+
|
200
|
+
def timeout_for (task, timeout)
|
201
|
+
unless @timeout
|
202
|
+
spawn_timeout_thread
|
203
|
+
end
|
204
|
+
|
205
|
+
@mutex.synchronize {
|
206
|
+
@timeouts[task] = timeout
|
207
|
+
|
208
|
+
wake_up_timeout
|
209
|
+
}
|
210
|
+
end
|
211
|
+
|
212
|
+
def shutdown_after (timeout)
|
213
|
+
Thread.new {
|
214
|
+
sleep timeout
|
215
|
+
|
216
|
+
shutdown
|
217
|
+
}
|
218
|
+
|
219
|
+
self
|
220
|
+
end
|
221
|
+
|
222
|
+
def wake_up_timeout
|
223
|
+
if @pipes
|
224
|
+
@pipes.last.write_nonblock 'x' rescue nil
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
private
|
229
|
+
def spawn_thread
|
230
|
+
@spawned += 1
|
231
|
+
|
232
|
+
thread = Thread.new {
|
233
|
+
loop do
|
234
|
+
task = @mutex.synchronize {
|
235
|
+
if @todo.empty?
|
236
|
+
while @todo.empty?
|
237
|
+
if @trim_requests > 0
|
238
|
+
@trim_requests -= 1
|
239
|
+
|
240
|
+
break
|
241
|
+
end
|
242
|
+
|
243
|
+
break if shutdown?
|
244
|
+
|
245
|
+
@waiting += 1
|
246
|
+
@cond.wait @mutex
|
247
|
+
@waiting -= 1
|
248
|
+
|
249
|
+
break !shutdown?
|
250
|
+
end or break
|
251
|
+
end
|
252
|
+
|
253
|
+
@todo.shift
|
254
|
+
} or break
|
255
|
+
|
256
|
+
task.execute(thread)
|
257
|
+
|
258
|
+
break if @shutdown == :now
|
259
|
+
|
260
|
+
trim if auto_trim? && @spawned > @min
|
261
|
+
end
|
262
|
+
|
263
|
+
@mutex.synchronize {
|
264
|
+
@spawned -= 1
|
265
|
+
@workers.delete thread
|
266
|
+
}
|
267
|
+
}
|
268
|
+
|
269
|
+
@workers << thread
|
270
|
+
|
271
|
+
thread
|
272
|
+
end
|
273
|
+
|
274
|
+
def spawn_timeout_thread
|
275
|
+
@pipes = IO.pipe
|
276
|
+
@timeout = Thread.new {
|
277
|
+
loop do
|
278
|
+
now = Time.now
|
279
|
+
timeout = @timeouts.map {|task, timeout|
|
280
|
+
next unless task.started_at
|
281
|
+
|
282
|
+
now - task.started_at + task.timeout
|
283
|
+
}.compact.min unless @timeouts.empty?
|
284
|
+
|
285
|
+
readable, = IO.select([@pipes.first], nil, nil, timeout)
|
286
|
+
|
287
|
+
break if @shutdown == :now
|
288
|
+
|
289
|
+
if readable && !readable.empty?
|
290
|
+
readable.first.read_nonblock 1024
|
291
|
+
end
|
292
|
+
|
293
|
+
now = Time.now
|
294
|
+
@timeouts.each {|task, time|
|
295
|
+
next if !task.started_at || task.terminated? || task.finished?
|
296
|
+
|
297
|
+
if now > task.started_at + task.timeout
|
298
|
+
task.timeout!
|
299
|
+
end
|
300
|
+
}
|
301
|
+
|
302
|
+
@timeouts.reject! { |task, _| task.terminated? || task.finished? }
|
303
|
+
|
304
|
+
break if @shutdown == :now
|
305
|
+
end
|
306
|
+
}
|
307
|
+
end
|
308
|
+
end
|