vtools 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/INSTALL +0 -0
- data/LICENSE +20 -0
- data/README.md +131 -0
- data/Rakefile +29 -0
- data/bin/vtools +22 -0
- data/doc/CONFIG.md +36 -0
- data/doc/HOOKS.md +37 -0
- data/doc/LIB_EXAMPLE.md +109 -0
- data/extconf.rb +7 -0
- data/lib/vtools.rb +79 -0
- data/lib/vtools/config.rb +91 -0
- data/lib/vtools/convert_options.rb +155 -0
- data/lib/vtools/converter.rb +98 -0
- data/lib/vtools/errors.rb +21 -0
- data/lib/vtools/handler.rb +43 -0
- data/lib/vtools/harvester.rb +71 -0
- data/lib/vtools/job.rb +48 -0
- data/lib/vtools/options.rb +101 -0
- data/lib/vtools/shared_methods.rb +131 -0
- data/lib/vtools/storage.rb +67 -0
- data/lib/vtools/thumbnailer.rb +93 -0
- data/lib/vtools/thumbs_options.rb +80 -0
- data/lib/vtools/version.rb +6 -0
- data/lib/vtools/version.rb~ +4 -0
- data/lib/vtools/video.rb +158 -0
- data/setup.rb +1585 -0
- data/spec/config_spec.rb +142 -0
- data/spec/convert_options_spec.rb +284 -0
- data/spec/converter_spec.rb +167 -0
- data/spec/errors_spec.rb +39 -0
- data/spec/fixtures/outputs/file_with_iso-8859-1.txt +35 -0
- data/spec/fixtures/outputs/file_with_no_audio.txt +18 -0
- data/spec/fixtures/outputs/file_with_non_supported_audio.txt +29 -0
- data/spec/fixtures/outputs/file_with_start_value.txt +19 -0
- data/spec/fixtures/outputs/file_with_surround_sound.txt +19 -0
- data/spec/handler_spec.rb +81 -0
- data/spec/harvester_spec.rb +189 -0
- data/spec/job_spec.rb +130 -0
- data/spec/options_spec.rb +52 -0
- data/spec/shared_methods_spec.rb +351 -0
- data/spec/spec_helper.rb +20 -0
- data/spec/storage_spec.rb +106 -0
- data/spec/thumbnailer_spec.rb +178 -0
- data/spec/thumbs_options_spec.rb +159 -0
- data/spec/video_spec.rb +274 -0
- data/vtools.gemspec +29 -0
- metadata +177 -0
data/INSTALL
ADDED
File without changes
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2011 Vadim Todorov
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
# VTools
|
2
|
+
|
3
|
+
Daemon video processor tools to operate the video (get info, encode & generate thumbnails).
|
4
|
+
Under the hood ffmpeg & ffmpegthumbnailer are used.
|
5
|
+
Some ideas has been taken at the streamio-ffmpeg gem (parse output methods).
|
6
|
+
|
7
|
+
Project was developed for the [WebTV](http://web.tv)
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
(sudo) gem install vtools
|
12
|
+
|
13
|
+
Please read changelog to check ffmpeg versions compatibility.
|
14
|
+
|
15
|
+
## Usage
|
16
|
+
|
17
|
+
### Getting started
|
18
|
+
|
19
|
+
Before start, daemon should be correctly configured, to have valid access to the storage.
|
20
|
+
Mandatory methods are: **connect**, **recv** and **send**.
|
21
|
+
|
22
|
+
``` ruby
|
23
|
+
#--file library.rb--#
|
24
|
+
# encoding: binary
|
25
|
+
|
26
|
+
#to setup storage:
|
27
|
+
VTools::Storage.setup do
|
28
|
+
|
29
|
+
# connection setup
|
30
|
+
connect_action do
|
31
|
+
# ... connect to the storage (persistent)
|
32
|
+
end
|
33
|
+
|
34
|
+
# message reciever
|
35
|
+
# should return JSON encoded string
|
36
|
+
# see complete storage setup reference for details
|
37
|
+
recv_action do
|
38
|
+
# ... job data recieve algorithm
|
39
|
+
end
|
40
|
+
|
41
|
+
# message sender
|
42
|
+
# recieves hash: { :data => execution_result, :action => executed_action }
|
43
|
+
# execution_result can be video object or array with thumbnails
|
44
|
+
send_action do |result|
|
45
|
+
# ... send action here
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# storage can be setup separate
|
50
|
+
VTools::Storage.connect_action do
|
51
|
+
# ... connect to the storage
|
52
|
+
end
|
53
|
+
```
|
54
|
+
|
55
|
+
### Setup message (JSON)
|
56
|
+
|
57
|
+
```
|
58
|
+
{ "action" : "convert|thumbs|info", "file" : "path/to/file", "setup" : < setup > }
|
59
|
+
# setup can be:
|
60
|
+
# -- "predefined_set_id"
|
61
|
+
# -- { ffmpeg_options_hash }
|
62
|
+
# -- { "set": "predefined_set_str", ffmpeg_options_hash }
|
63
|
+
```
|
64
|
+
|
65
|
+
### User friendly option names
|
66
|
+
|
67
|
+
```
|
68
|
+
converter (ffmpeg)
|
69
|
+
preserve_aspect (true or false)
|
70
|
+
extension (result file extension)
|
71
|
+
width, height
|
72
|
+
resolution
|
73
|
+
duration
|
74
|
+
|
75
|
+
thumbnailer (ffmpegthumbnailer)
|
76
|
+
thumb_count
|
77
|
+
thumb_start_point (in percents)
|
78
|
+
time (time offset, alias for -t)
|
79
|
+
quality (0 - 10)
|
80
|
+
width
|
81
|
+
```
|
82
|
+
|
83
|
+
## Start
|
84
|
+
|
85
|
+
To launch daemon is enough to require library with storage setup:
|
86
|
+
(sudo) vtools start -- -r library
|
87
|
+
|
88
|
+
## Options
|
89
|
+
|
90
|
+
### Daemon options are
|
91
|
+
start
|
92
|
+
stop
|
93
|
+
restart
|
94
|
+
|
95
|
+
### Application options are:
|
96
|
+
-c or --config-file - load config from file
|
97
|
+
-r or --require - load ruby library file (can be used more than once)
|
98
|
+
|
99
|
+
### To see complete options list use
|
100
|
+
vtools --help
|
101
|
+
|
102
|
+
### Using logger
|
103
|
+
|
104
|
+
By default the `logger` gem is used. But there is possibility to set custom logger, that is compatible with the default logger.
|
105
|
+
|
106
|
+
``` ruby
|
107
|
+
VTools.logger = CustomLoger.new($stdout)
|
108
|
+
```
|
109
|
+
|
110
|
+
### Additioinal methods
|
111
|
+
|
112
|
+
Path generator is used by the thumnailer, converter or both to generate necessary dir tree logic for the media.
|
113
|
+
It accepts file name and should return relative path (excluding file name itself)
|
114
|
+
|
115
|
+
``` ruby
|
116
|
+
# path generator (used to )
|
117
|
+
VTools.path_generator do |file_name|
|
118
|
+
# ..
|
119
|
+
end
|
120
|
+
|
121
|
+
```
|
122
|
+
|
123
|
+
**Network calls** (TCP GET request, that will return message body content, ignoring response headers)
|
124
|
+
|
125
|
+
``` ruby
|
126
|
+
|
127
|
+
# http calls
|
128
|
+
VTools.network_call "site.com/some/uri"
|
129
|
+
VTools.network_call "www.site.com"
|
130
|
+
VTools.network_call "http://www.site.com"
|
131
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
|
3
|
+
require 'rake/gempackagetask'
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift 'lib'
|
6
|
+
require "vtools/version"
|
7
|
+
|
8
|
+
spec = Gem::Specification.new do |s|
|
9
|
+
s.name = "vtools"
|
10
|
+
s.summary = "Daemon tools to operate the video (get info, encode & generate thumbnails)."
|
11
|
+
s.description = File.read(File.join(File.dirname(__FILE__), 'README.md'))
|
12
|
+
s.extensions = 'extconf.rb'
|
13
|
+
s.version = VTools::VERSION.join('.')
|
14
|
+
s.requirements = ['ffmpeg v >= 1.8', 'ffmpegthumbnailer v >= 2', 'gem Daemons v >= 1.1.4']
|
15
|
+
s.author = "tofir"
|
16
|
+
s.email = "v.tofir@gmail.com"
|
17
|
+
s.homepage = "http://tofir.comuv.com"
|
18
|
+
s.platform = Gem::Platform::RUBY
|
19
|
+
s.required_ruby_version = ">=1.9"
|
20
|
+
s.files = Dir['**/**']
|
21
|
+
s.executables = ["vtools"]
|
22
|
+
s.test_files = Dir["spec/*_spec.rb"]
|
23
|
+
|
24
|
+
s.add_dependency 'daemons', '>= 1.1.4'
|
25
|
+
s.add_dependency 'json'
|
26
|
+
s.add_development_dependency 'rspec'
|
27
|
+
end
|
28
|
+
|
29
|
+
Rake::GemPackageTask.new(spec).define
|
data/bin/vtools
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- encoding: binary -*-
|
3
|
+
|
4
|
+
require 'daemons'
|
5
|
+
require 'vtools'
|
6
|
+
|
7
|
+
VTools::Options.parse! ARGV
|
8
|
+
|
9
|
+
options = {:multiple => true}
|
10
|
+
|
11
|
+
if VTools::CONFIG[:logging]
|
12
|
+
options[:backtrace] = true
|
13
|
+
options[:log_output] = true
|
14
|
+
options[:log_dir] = VTools::CONFIG[:PWD]
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
# daemonize server
|
19
|
+
Daemons.run_proc 'harvester', options do
|
20
|
+
# start process
|
21
|
+
VTools::Harvester.daemonize!
|
22
|
+
end
|
data/doc/CONFIG.md
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# Using config file (YAML)
|
2
|
+
|
3
|
+
To make VTools parse config file it should be given to it with -c option:
|
4
|
+
|
5
|
+
$ vtools start -- -c config.json
|
6
|
+
|
7
|
+
|
8
|
+
## Format
|
9
|
+
|
10
|
+
|
11
|
+
#system
|
12
|
+
:ffmpeg_binary: 'ffmpeg'
|
13
|
+
:thumb_binary: 'ffmpegthumbnailer'
|
14
|
+
:library: ['lib.rb']
|
15
|
+
|
16
|
+
#harvester
|
17
|
+
:max_jobs: 5
|
18
|
+
:harvester_timer: 3 # timeout between job requests
|
19
|
+
:temp_dir: "/tmp"
|
20
|
+
|
21
|
+
#converter
|
22
|
+
:validate_duration: false
|
23
|
+
:video_storage: "/home/projects/video_storage"
|
24
|
+
|
25
|
+
#thumbnailer
|
26
|
+
:thumb_storage: "/home/projects/thumb_storage"
|
27
|
+
|
28
|
+
#predefined video qualities
|
29
|
+
:video_set:
|
30
|
+
# SET_NAME -vcodec -acodec -s WDTxHGT -vb -ab -ar -ac EXT POSTFIX [-vpre]
|
31
|
+
:x264_180p: [ 'libx264', 'libfaac', '240x180', '96k', '64k', 22050, 2, 'mp4', '_180', 'normal' ]
|
32
|
+
|
33
|
+
# predefined thumbnailer setup
|
34
|
+
:thumb_set:
|
35
|
+
# SET_NAME -s -q count start%
|
36
|
+
:w120s: [ 120, 10, 5, 0 ]
|
data/doc/HOOKS.md
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Using hooks
|
2
|
+
|
3
|
+
Multiple actions can be attached for a single hook.
|
4
|
+
|
5
|
+
**hooks placeholders:**
|
6
|
+
|
7
|
+
* job_started (video, action)
|
8
|
+
* job_finished (result, video, action)
|
9
|
+
|
10
|
+
* before_convert (video, command)
|
11
|
+
* in_convert (video, progress)
|
12
|
+
* convert_success (video, output_file)
|
13
|
+
* convert_error (video, error, ffmpeg_output)
|
14
|
+
|
15
|
+
* before_thumb (video, config)
|
16
|
+
* in_thumb (video, thumb)
|
17
|
+
* thumb_success (video, thumbs_array)
|
18
|
+
* thumb_error (video, errors)
|
19
|
+
|
20
|
+
``` ruby
|
21
|
+
# multiple hooks setup
|
22
|
+
VTools::Handler.collection do
|
23
|
+
set :in_convert do |video|
|
24
|
+
#..
|
25
|
+
end
|
26
|
+
|
27
|
+
set :in_thumb do |video|
|
28
|
+
#..
|
29
|
+
end
|
30
|
+
# ...
|
31
|
+
end
|
32
|
+
|
33
|
+
# single hook setup
|
34
|
+
VTools::Handler.set :job_started do
|
35
|
+
#..
|
36
|
+
end
|
37
|
+
```
|
data/doc/LIB_EXAMPLE.md
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
# Library example
|
2
|
+
|
3
|
+
Library can be required via `-r` option
|
4
|
+
|
5
|
+
$ vtools start -- -r library
|
6
|
+
$ vtools start -- -r library.rb
|
7
|
+
|
8
|
+
```ruby
|
9
|
+
# -*- encoding: binary -*-
|
10
|
+
|
11
|
+
# path generator
|
12
|
+
VTools.path_generator do |file_name|
|
13
|
+
|
14
|
+
storage_path = VTools::CONFIG[:PWD] + "/../test_storage"
|
15
|
+
first = file_name[0...2]
|
16
|
+
second = file_name[2...4]
|
17
|
+
# create all parent dirs (storage... path .. ) ???
|
18
|
+
if (error = `mkdir -p #{storage_path}/#{first}/#{second}`).empty?
|
19
|
+
"#{storage_path}/#{first}/#{second}"
|
20
|
+
else
|
21
|
+
raise Exception, "Can't create storage dirs (#{error})"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# storage setup
|
26
|
+
VTools::Storage.setup do
|
27
|
+
|
28
|
+
connect_action do
|
29
|
+
@mq = MQ.new
|
30
|
+
print "----------- connected to the MQ -------\n"
|
31
|
+
end
|
32
|
+
|
33
|
+
# recieve data for processing
|
34
|
+
recv_action do
|
35
|
+
job = @mq.recv
|
36
|
+
setup = "{\"set\" : \"flv_240p\", \"acodec\" : \"#{job}\" }"
|
37
|
+
"{\"action\" : \"#{action}\", \"file\" : \"#{file}\", \"setup\" : #{setup} }"
|
38
|
+
end
|
39
|
+
|
40
|
+
send_action do |result|
|
41
|
+
@mq.send "video.success #{result[:action]} "
|
42
|
+
print "----------------> 'terminated' MQ sent (#{result[:data].to_json}) - action #{result[:action]}\n"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# hooks
|
47
|
+
VTools::Handler.collection do
|
48
|
+
|
49
|
+
set :job_started do |video, action|
|
50
|
+
print "(#{video.name}) ------> job | started | {#{video}} :: scope: #{action}\n"
|
51
|
+
end
|
52
|
+
|
53
|
+
set :in_convert do |video, status|
|
54
|
+
|
55
|
+
# cgeate thumb each step
|
56
|
+
current = (status * 1000).round / 10.0
|
57
|
+
@count ||= 0;
|
58
|
+
|
59
|
+
# generate thumbs on the fly
|
60
|
+
if @count == 10
|
61
|
+
@count = 0
|
62
|
+
print "(#{video.name}) >>>>>>> add_thumb --------- #{current}%\n"
|
63
|
+
video.create_thumbs :thumb_count => 1,
|
64
|
+
:time => current.to_i,
|
65
|
+
:postfix => current.to_i,
|
66
|
+
:width => 600
|
67
|
+
end
|
68
|
+
|
69
|
+
@count += 1
|
70
|
+
|
71
|
+
print "(#{video.name}) +++++++ convert | status | (#{current}%)\n"
|
72
|
+
end
|
73
|
+
|
74
|
+
set :in_thumb do |video, thumb|
|
75
|
+
print "(#{video.name}) +++++++ generate| thumbs | (#{thumb})\n"
|
76
|
+
end
|
77
|
+
|
78
|
+
set :before_convert do |video, command|
|
79
|
+
print "(#{video.name}) ------- convert |scheduled| {#{video}}\n"
|
80
|
+
end
|
81
|
+
|
82
|
+
set :before_thumb do |video, config|
|
83
|
+
print "(#{video.name}) ------- thumbs |scheduled| {#{video}} :: (config: #{config})\n"
|
84
|
+
end
|
85
|
+
|
86
|
+
set :convert_error do |video|
|
87
|
+
print "(#{video.name}) !!!!!!! convert | failed | {#{video}}\n"
|
88
|
+
end
|
89
|
+
|
90
|
+
set :convert_success do |video|
|
91
|
+
print "(#{video.name}) <------ convert |finished | {#{video}}\n"
|
92
|
+
end
|
93
|
+
|
94
|
+
set :thumb_success do |video, thumbs|
|
95
|
+
if thumbs
|
96
|
+
images = ":: "
|
97
|
+
thumbs.each do |hash|
|
98
|
+
images += "#{hash[:offset]}||"
|
99
|
+
end
|
100
|
+
images = "#{images} count:#{thumbs.count}"
|
101
|
+
end
|
102
|
+
print "(#{video.name}) <------ thumbs |finished | {#{video}} #{images}\n"
|
103
|
+
end
|
104
|
+
|
105
|
+
set :job_finished do |result, video, action|
|
106
|
+
print "(#{video.name}) <------ job |finished | {#{video}} :: scope: #{action}\n"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
```
|
data/extconf.rb
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
|
3
|
+
# validate if ffmpeg & ffmpegthumbnails exists
|
4
|
+
ffmpeg = `which ffmpeg`
|
5
|
+
thumbnailer = `which ffmpegthumbnailer`
|
6
|
+
|
7
|
+
puts "\n*---------------> Could not find ffmpeg/ffmpegthumbnailer.. <---------------*\n" if thumbnailer.empty? || ffmpeg.empty?
|
data/lib/vtools.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
|
3
|
+
module VTools
|
4
|
+
|
5
|
+
# set global $LOAD_PATH
|
6
|
+
$: << File.dirname(__FILE__) << Dir.getwd
|
7
|
+
|
8
|
+
Thread.abort_on_exception = true
|
9
|
+
|
10
|
+
require 'optparse'
|
11
|
+
require 'json'
|
12
|
+
require 'yaml'
|
13
|
+
require 'ostruct'
|
14
|
+
require 'open3'
|
15
|
+
require 'logger'
|
16
|
+
|
17
|
+
require 'vtools/version'
|
18
|
+
require 'vtools/errors'
|
19
|
+
require 'vtools/shared_methods'
|
20
|
+
require 'vtools/config'
|
21
|
+
require 'vtools/options'
|
22
|
+
require 'vtools/handler'
|
23
|
+
require 'vtools/storage'
|
24
|
+
require 'vtools/harvester'
|
25
|
+
require 'vtools/job'
|
26
|
+
require 'vtools/video'
|
27
|
+
require 'vtools/converter'
|
28
|
+
require 'vtools/thumbnailer'
|
29
|
+
require 'vtools/convert_options'
|
30
|
+
require 'vtools/thumbs_options'
|
31
|
+
|
32
|
+
include SharedMethods # extend with common methods
|
33
|
+
|
34
|
+
# callbacks after ARGV config has been parsed
|
35
|
+
Handler.collection do
|
36
|
+
|
37
|
+
# load external config file & external libs
|
38
|
+
set :config_parsed do
|
39
|
+
CONFIG.load!
|
40
|
+
load_libs
|
41
|
+
end
|
42
|
+
|
43
|
+
# set job finished handlers
|
44
|
+
set :job_finished do |result, video, action|
|
45
|
+
# log job status
|
46
|
+
log :info, "Job finished. name: #{video.name}, action : #{action}"
|
47
|
+
end
|
48
|
+
|
49
|
+
# set converter error handlers
|
50
|
+
set :before_convert do |video, command|
|
51
|
+
# log status
|
52
|
+
log :info, "Running encoding... (#{video.name}) : #{command}"
|
53
|
+
end
|
54
|
+
|
55
|
+
# set converter error handlers
|
56
|
+
set :convert_success do |video, output_file|
|
57
|
+
# log status
|
58
|
+
log :info, "Encoding of #{video.path} to #{output_file} succeeded"
|
59
|
+
end
|
60
|
+
|
61
|
+
# set converter error handlers
|
62
|
+
set :convert_error do |video, errors, output|
|
63
|
+
# log error
|
64
|
+
log :error, "Failed encoding... #{video} #{output} #{errors}"
|
65
|
+
end
|
66
|
+
|
67
|
+
# set thumbnailer success handlers
|
68
|
+
set :thumb_success do |video, thumbs|
|
69
|
+
# log status
|
70
|
+
log :info, "Thumbnail creation of #{video.path} succeeded"
|
71
|
+
end
|
72
|
+
|
73
|
+
# set thumbnailer error handlers
|
74
|
+
set :thumb_error do |video, errors|
|
75
|
+
# log error
|
76
|
+
log :error, "Failed create thumbnail for the video #{video.name} #{errors}"
|
77
|
+
end
|
78
|
+
end # Handler.collection
|
79
|
+
end # VTools
|