vidibus-watch_folder 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.md +197 -0
- data/Rakefile +17 -0
- data/lib/generators/vidibus/templates/script +8 -0
- data/lib/generators/vidibus/watch_folder_generator.rb +14 -0
- data/lib/vidibus/watch_folder/base.rb +197 -0
- data/lib/vidibus/watch_folder/capistrano/recipes.rb +40 -0
- data/lib/vidibus/watch_folder/capistrano.rb +12 -0
- data/lib/vidibus/watch_folder/daemon.rb +46 -0
- data/lib/vidibus/watch_folder/job.rb +39 -0
- data/lib/vidibus/watch_folder/railtie.rb +7 -0
- data/lib/vidibus/watch_folder/util/directory.rb +17 -0
- data/lib/vidibus/watch_folder/util.rb +1 -0
- data/lib/vidibus/watch_folder/version.rb +5 -0
- data/lib/vidibus/watch_folder.rb +84 -0
- data/lib/vidibus-watch_folder.rb +1 -0
- metadata +256 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 André Pankratz
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,197 @@
|
|
1
|
+
# Vidibus::WatchFolder [![](http://travis-ci.org/vidibus/vidibus-watch_folder.png)](http://travis-ci.org/vidibus/vidibus-watch_folder)
|
2
|
+
|
3
|
+
This gem lets you create multipe watch folders within your application, e.g. to provide individual FTP mount points for customers, maybe in combination with [Vidibus::Pureftpd](https://github.com/vidibus/vidibus-pureftpd).
|
4
|
+
|
5
|
+
To store each watch folder configuration, [Mongoid](http://mongoid.org/en/mongoid/index.html) (~> 2.5) is used. Files are processed asynchronously with [DelayedJob](https://github.com/collectiveidea/delayed_job).
|
6
|
+
|
7
|
+
This gem is part of [Vidibus](http://vidibus.org), an open source toolset for building distributed (video) applications.
|
8
|
+
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
Add `gem 'vidibus-watch_folder'` to the Gemfile of your application. Then call `bundle install` on your console.
|
13
|
+
|
14
|
+
This gem relies on [Listen](https://github.com/guard/listen) to detect changes. If you're on Windows, you'll want to install an additional file system adapter to increase performance:
|
15
|
+
|
16
|
+
```ruby
|
17
|
+
# Windows only!
|
18
|
+
gem 'wdm', '~> 0.0.3'
|
19
|
+
```
|
20
|
+
|
21
|
+
|
22
|
+
## Usage
|
23
|
+
|
24
|
+
### Models
|
25
|
+
|
26
|
+
Setting up a custom watch folder model is easy. Since it's a `Mongoid::Document`, all of the `ActiveModel` magic is at your disposal. Just add some watch folder settings:
|
27
|
+
|
28
|
+
```ruby
|
29
|
+
class Example < Vidibus::WatchFolder::Base
|
30
|
+
|
31
|
+
# Define a root directory to store files in.
|
32
|
+
root Rails.root.join('examples')
|
33
|
+
|
34
|
+
# Define folders that should automatically be created.
|
35
|
+
folders 'in', 'out'
|
36
|
+
|
37
|
+
# Define callbacks to perform when files change.
|
38
|
+
#
|
39
|
+
# Use filter :when to define events to watch. Supported event types are:
|
40
|
+
# :added, :modified, :removed
|
41
|
+
#
|
42
|
+
# Add filter :delay to perform callback later. Execution will be delayed
|
43
|
+
# until the watched file will not have been changed for given period of time.
|
44
|
+
# This is useful for waiting until an upload is completed.
|
45
|
+
#
|
46
|
+
# Set filter :ignore to exclude file names matching given regex.
|
47
|
+
#
|
48
|
+
# Provide :folders to limit this callback to certain folders.
|
49
|
+
callback :create_upload, {
|
50
|
+
:when => :added,
|
51
|
+
:delay => 1.minute,
|
52
|
+
:folders => 'in',
|
53
|
+
:ignore => /^\.pureftpd-upload/
|
54
|
+
}
|
55
|
+
callback :destroy_upload, :when => :removed
|
56
|
+
|
57
|
+
# Callback to process created files
|
58
|
+
def create_upload(event, path)
|
59
|
+
...
|
60
|
+
end
|
61
|
+
|
62
|
+
# Callback to handle deleted files
|
63
|
+
def destroy_upload(event, path)
|
64
|
+
...
|
65
|
+
end
|
66
|
+
end
|
67
|
+
```
|
68
|
+
|
69
|
+
|
70
|
+
### Instances
|
71
|
+
|
72
|
+
Handling a watch folder instance is straightforward:
|
73
|
+
|
74
|
+
```ruby
|
75
|
+
example = Example.create
|
76
|
+
|
77
|
+
# Access instance properties
|
78
|
+
example.uuid # => 98fe6010e7b5012f7e4c6c626d58b44c
|
79
|
+
example.path # => '/path/to/rails/examples/98fe6010e7b5012f7e4c6c626d58b44c/'
|
80
|
+
example.files # => ['<FILE_PATH>', ...]
|
81
|
+
|
82
|
+
# Destroy the instance (will remove its path, too)
|
83
|
+
example.destroy
|
84
|
+
```
|
85
|
+
|
86
|
+
|
87
|
+
### Listening for file changes
|
88
|
+
|
89
|
+
File changes are detected by performing `Vidibus::WatchFolder.listen`. Beware, this method is blocking, so better spawn the daemon.
|
90
|
+
|
91
|
+
|
92
|
+
#### Listener daemon
|
93
|
+
|
94
|
+
To run the listener as daemon, this gem provides a shell script. Install it with
|
95
|
+
|
96
|
+
```
|
97
|
+
rails g vidibus:watch_folder
|
98
|
+
```
|
99
|
+
|
100
|
+
The daemon requires that `gem 'daemons'` is installed. To spawn him, enter
|
101
|
+
|
102
|
+
```
|
103
|
+
script/watch_folder start
|
104
|
+
```
|
105
|
+
|
106
|
+
#### Possible caveat
|
107
|
+
|
108
|
+
To collect the paths to listen to, `Vidibus::WatchFolder.listen` requires that all classes inheriting `Vidibus::WatchFolder::Base` have been loaded.
|
109
|
+
|
110
|
+
Because Rails is autoloading almost everything in development, this requirement is not met without the help of a little hack: To trigger autoloading, the listener collects all aforementioned class names from the `app` directory and constantizes them.
|
111
|
+
|
112
|
+
**So here's the caveat:** If you define watch folder models outside of the `app` directory, you'll have to let the listener know. An initializer is perfect for that:
|
113
|
+
|
114
|
+
```ruby
|
115
|
+
# Collect all watch folder models in lib, too
|
116
|
+
Vidibus::WatchFolder.autoload_paths << '/lib/**/*.rb'
|
117
|
+
```
|
118
|
+
|
119
|
+
|
120
|
+
## Deployment
|
121
|
+
|
122
|
+
A Capistrano configuration is included. Require it in your Capistrano `config.rb`.
|
123
|
+
|
124
|
+
```ruby
|
125
|
+
require 'vidibus/watch_folder/capistrano'
|
126
|
+
```
|
127
|
+
|
128
|
+
That will add a bunch of callback hooks.
|
129
|
+
|
130
|
+
```ruby
|
131
|
+
after 'deploy:stop', 'vidibus:watch_folder:stop'
|
132
|
+
after 'deploy:start', 'vidibus:watch_folder:start'
|
133
|
+
after 'deploy:restart', 'vidibus:watch_folder:restart'
|
134
|
+
```
|
135
|
+
|
136
|
+
If you need more control over the callbacks, you may load just the recipes without the hooks.
|
137
|
+
|
138
|
+
```ruby
|
139
|
+
require 'vidibus/watch_folder/capistrano/recipes'
|
140
|
+
```
|
141
|
+
|
142
|
+
### Shared folders
|
143
|
+
|
144
|
+
In case you want to put files into a shared folder, you may run into a validation issue. Here's a configuration for our watch folder example that gets symlinked with a twist:
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
namespace :examples do
|
148
|
+
task :setup do
|
149
|
+
path = File.join(shared_path, 'examples')
|
150
|
+
run "mkdir -p #{path}"
|
151
|
+
run "chmod -R 777 #{path}"
|
152
|
+
end
|
153
|
+
|
154
|
+
task :symlink do
|
155
|
+
run "ln -nfs #{shared_path}/examples #{release_path}/"
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
after 'deploy:setup', 'examples:setup'
|
160
|
+
before 'deploy:assets:precompile', 'examples:symlink'
|
161
|
+
```
|
162
|
+
|
163
|
+
The important thing is the last line. Instead of the usual `after 'deploy:update_code'` hook we're triggering the symlink on `before 'deploy:assets:precompile'`. The reason is that precompiling initializes the Rails app which will fail if the example directory does not exist yet.
|
164
|
+
|
165
|
+
|
166
|
+
## Testing
|
167
|
+
|
168
|
+
To test this gem, call `bundle install` and `bundle exec rspec spec` on your console.
|
169
|
+
|
170
|
+
When testing your application you may want to define a different root path for your watch folder models. Just override them somewhere in your test files, for example in `spec_helper.rb`:
|
171
|
+
|
172
|
+
```ruby
|
173
|
+
# Set different root for watch folder example
|
174
|
+
Example.root('spec/support/examples')
|
175
|
+
```
|
176
|
+
|
177
|
+
Make sure that directory exists. From your Rails root call:
|
178
|
+
|
179
|
+
```
|
180
|
+
mkdir spec/support/examples
|
181
|
+
touch spec/support/examples/.gitkeep
|
182
|
+
```
|
183
|
+
|
184
|
+
To clean up the test folders, add the following to your RSpec config:
|
185
|
+
|
186
|
+
```ruby
|
187
|
+
RSpec.configure do |config|
|
188
|
+
config.before(:each) do
|
189
|
+
FileUtils.rm_r(Dir['spec/support/examples/*'].reject {|e| e == '.gitkeep'})
|
190
|
+
end
|
191
|
+
end
|
192
|
+
```
|
193
|
+
|
194
|
+
|
195
|
+
## Copyright
|
196
|
+
|
197
|
+
© 2012 André Pankratz. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
$:.unshift File.expand_path('../lib/', __FILE__)
|
2
|
+
|
3
|
+
require 'bundler'
|
4
|
+
require 'rdoc/task'
|
5
|
+
require 'rspec'
|
6
|
+
require 'rspec/core/rake_task'
|
7
|
+
require 'vidibus/watch_folder/version'
|
8
|
+
|
9
|
+
Bundler::GemHelper.install_tasks
|
10
|
+
|
11
|
+
Rake::RDocTask.new do |rdoc|
|
12
|
+
rdoc.rdoc_dir = 'rdoc'
|
13
|
+
rdoc.title = "vidibus-sysinfo #{Vidibus::WatchFolder::VERSION}"
|
14
|
+
rdoc.rdoc_files.include('README*')
|
15
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
16
|
+
rdoc.options << '--charset=utf-8'
|
17
|
+
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require File.expand_path('../../config/environment', __FILE__)
|
4
|
+
require 'vidibus/watch_folder/daemon'
|
5
|
+
|
6
|
+
Vidibus::WatchFolder.autoload_paths << Rails.root.join('app/**/*.rb')
|
7
|
+
Vidibus::WatchFolder.path_mapping << [Rails.root.to_s, '.+']
|
8
|
+
Vidibus::WatchFolder::Daemon.new(ARGV).daemonize
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
require 'rails/generators/named_base'
|
3
|
+
|
4
|
+
module Vidibus
|
5
|
+
class WatchFolderGenerator < Rails::Generators::Base
|
6
|
+
|
7
|
+
self.source_paths << File.join(File.dirname(__FILE__), 'templates')
|
8
|
+
|
9
|
+
def create_script_file
|
10
|
+
template 'script', 'script/watch_folder'
|
11
|
+
chmod 'script/watch_folder', 0755
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,197 @@
|
|
1
|
+
require 'mongoid'
|
2
|
+
require 'digest'
|
3
|
+
require 'vidibus-uuid'
|
4
|
+
|
5
|
+
module Vidibus
|
6
|
+
module WatchFolder
|
7
|
+
class Base
|
8
|
+
include Mongoid::Document
|
9
|
+
include Vidibus::Uuid::Mongoid
|
10
|
+
|
11
|
+
class ConfigError < Error; end
|
12
|
+
|
13
|
+
after_create :setup
|
14
|
+
after_destroy :teardown
|
15
|
+
|
16
|
+
# Return the configured root path
|
17
|
+
def root
|
18
|
+
self.class.config[:root] || raise(ConfigError, 'No root configured')
|
19
|
+
end
|
20
|
+
|
21
|
+
# Return the absolute path to this watch folder.
|
22
|
+
def path
|
23
|
+
File.join(root, uuid) if uuid
|
24
|
+
end
|
25
|
+
|
26
|
+
# Return a list of file paths within this watch folder.
|
27
|
+
def files
|
28
|
+
Dir["#{path}/**/*"].reject do |entry|
|
29
|
+
File.directory?(entry)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Handle event for given file path.
|
34
|
+
# Unless a checksum is provided, a asynchronous job will be started.
|
35
|
+
# If provided checksum matches the current one (or if execution
|
36
|
+
# shall not be delayed), appropriate callbacks will be performed.
|
37
|
+
def handle(event, file_path, last_checksum = nil)
|
38
|
+
return unless File.exist?(file_path) && !File.directory?(file_path)
|
39
|
+
callbacks = self.class.config[:callback]
|
40
|
+
callbacks.each do |folder, handlers|
|
41
|
+
unless folder == :any
|
42
|
+
pattern = %r(^#{path}/#{folder}/.+$)
|
43
|
+
next unless file_path[pattern]
|
44
|
+
end
|
45
|
+
matching = handlers.select { |c| c[:when].include?(event) }
|
46
|
+
matching.each do |handler|
|
47
|
+
next if handler[:ignore] && file_path.match(handler[:ignore])
|
48
|
+
checksum ||= Vidibus::WatchFolder.checksum(file_path)
|
49
|
+
delay = handler[:delay]
|
50
|
+
if checksum == last_checksum || (last_checksum && !delay)
|
51
|
+
send(handler[:method], event, file_path)
|
52
|
+
else
|
53
|
+
Job.create(uuid, event, file_path, checksum, delay)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class << self
|
60
|
+
|
61
|
+
# Set root path of this kind of watch folder.
|
62
|
+
def root(path)
|
63
|
+
path = File.expand_path(path)
|
64
|
+
unless Util::Directory.valid?(path)
|
65
|
+
raise ConfigError, "Given root '#{path}' must be a read and writable folder"
|
66
|
+
end
|
67
|
+
unless Vidibus::WatchFolder.roots.include?(path)
|
68
|
+
Vidibus::WatchFolder.roots << path
|
69
|
+
end
|
70
|
+
config[:root] = path
|
71
|
+
end
|
72
|
+
|
73
|
+
# Define folders to create automatically when an instance of this
|
74
|
+
# kind of watch folder is created.
|
75
|
+
def folders(*args)
|
76
|
+
raise ConfigError, 'Define folders' unless args.any?
|
77
|
+
config[:folders] = string_list(args)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Define callbacks to perform when files change.
|
81
|
+
#
|
82
|
+
# Add filter :when to define events to watch. Supported event types
|
83
|
+
# are :added, :modified, an :removed
|
84
|
+
#
|
85
|
+
# Add filter :delay to perform callback later. Execution will then be
|
86
|
+
# delayed until the watched file will not have been changed for given
|
87
|
+
# period of time. This is useful for waiting until an upload is
|
88
|
+
# completed. If no delay has been configured, execution will be
|
89
|
+
# asynchronous nonetheless.
|
90
|
+
#
|
91
|
+
# Set filter :ignore to exclude file names matching given regex.
|
92
|
+
#
|
93
|
+
# Provide :folders to limit this callback to certain folders.
|
94
|
+
def callback(method, options = {})
|
95
|
+
config[:callback] ||= {}
|
96
|
+
opts = {:method => method}
|
97
|
+
if events = events_options(options)
|
98
|
+
opts[:when] = events
|
99
|
+
end
|
100
|
+
if delay = delay_options(options)
|
101
|
+
opts[:delay] = delay
|
102
|
+
end
|
103
|
+
if ignore = ignore_options(options)
|
104
|
+
opts[:ignore] = ignore
|
105
|
+
end
|
106
|
+
folders_options(options).each do |folder|
|
107
|
+
config[:callback][folder] ||= []
|
108
|
+
config[:callback][folder] << opts
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# Inheritable getter for config.
|
113
|
+
def config
|
114
|
+
@config ||= {}
|
115
|
+
end
|
116
|
+
|
117
|
+
# Inheritable setter for config.
|
118
|
+
def config=(value)
|
119
|
+
@config = value
|
120
|
+
end
|
121
|
+
|
122
|
+
# Find an instance of this kind of watch folder by its UUID.
|
123
|
+
# Will raise an exception if no instance can be found.
|
124
|
+
def find_by_uuid(uuid)
|
125
|
+
found = where(:uuid => uuid).first || begin
|
126
|
+
raise(Mongoid::Errors::DocumentNotFound.new(self, :uuid => uuid))
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
def string_list(input)
|
133
|
+
Array(input).map { |i| i.to_s }
|
134
|
+
end
|
135
|
+
|
136
|
+
def folders_options(options)
|
137
|
+
list = string_list(options.delete(:folders))
|
138
|
+
list = [:any] unless list.any?
|
139
|
+
list
|
140
|
+
end
|
141
|
+
|
142
|
+
def events_options(options)
|
143
|
+
return unless events = string_list(options.delete(:when))
|
144
|
+
if events.any?
|
145
|
+
if (events-EVENTS).any?
|
146
|
+
raise ConfigError, "Only these events are supported: #{EVENTS}"
|
147
|
+
end
|
148
|
+
return events
|
149
|
+
end
|
150
|
+
nil
|
151
|
+
end
|
152
|
+
|
153
|
+
def delay_options(options)
|
154
|
+
return unless delay = options.delete(:delay)
|
155
|
+
unless delay.is_a?(Integer) && delay > 0
|
156
|
+
raise ConfigError, 'Delay must be defined in seconds'
|
157
|
+
end
|
158
|
+
delay
|
159
|
+
end
|
160
|
+
|
161
|
+
def ignore_options(options)
|
162
|
+
return unless ignore = options.delete(:ignore)
|
163
|
+
unless ignore.is_a?(Regexp)
|
164
|
+
raise ConfigError, 'Ignore pattern must be a regular expression'
|
165
|
+
end
|
166
|
+
ignore
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
private
|
171
|
+
|
172
|
+
def setup
|
173
|
+
setup_path
|
174
|
+
setup_folders
|
175
|
+
end
|
176
|
+
|
177
|
+
def setup_path
|
178
|
+
FileUtils.mkdir_p(path)
|
179
|
+
end
|
180
|
+
|
181
|
+
def setup_folders
|
182
|
+
folders = self.class.config[:folders]
|
183
|
+
return unless folders
|
184
|
+
folders.each do |folder|
|
185
|
+
FileUtils.mkdir_p(File.join(path, folder))
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
def teardown
|
190
|
+
unless path.to_s.length > 5
|
191
|
+
raise ConfigError, "#{path} is too short! Exiting for security reasons."
|
192
|
+
end
|
193
|
+
FileUtils.rm_r(path) if File.exist?(path)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# Capistrano Recipes for watching folders.
|
2
|
+
#
|
3
|
+
# Load this file from your Capistrano config.rb:
|
4
|
+
# require 'vidibus/watch_folder/capistrano/recipes'
|
5
|
+
#
|
6
|
+
# Add these callbacks to have the watch_folder process restart when the server
|
7
|
+
# is restarted:
|
8
|
+
#
|
9
|
+
# after 'deploy:stop', 'vidibus:watch_folder:stop'
|
10
|
+
# after 'deploy:start', 'vidibus:watch_folder:start'
|
11
|
+
# after 'deploy:restart', 'vidibus:watch_folder:restart'
|
12
|
+
#
|
13
|
+
Capistrano::Configuration.instance.load do
|
14
|
+
namespace :vidibus do
|
15
|
+
namespace :watch_folder do
|
16
|
+
def rails_env
|
17
|
+
fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
|
18
|
+
end
|
19
|
+
|
20
|
+
def roles
|
21
|
+
fetch(:delayed_job_server_role, :app)
|
22
|
+
end
|
23
|
+
|
24
|
+
desc 'Stop the watch_folder process'
|
25
|
+
task :stop, :roles => lambda { roles } do
|
26
|
+
run "cd #{current_path};#{rails_env} script/watch_folder stop"
|
27
|
+
end
|
28
|
+
|
29
|
+
desc 'Start the watch_folder process'
|
30
|
+
task :start, :roles => lambda { roles } do
|
31
|
+
run "cd #{current_path};#{rails_env} script/watch_folder start"
|
32
|
+
end
|
33
|
+
|
34
|
+
desc 'Restart the watch_folder process'
|
35
|
+
task :restart, :roles => lambda { roles } do
|
36
|
+
run "cd #{current_path};#{rails_env} script/watch_folder restart"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'vidibus/watch_folder/capistrano/recipes'
|
2
|
+
|
3
|
+
# Run Capistrano Recipes for watching folders.
|
4
|
+
#
|
5
|
+
# Load this file from your Capistrano config.rb:
|
6
|
+
# require 'vidibus/watch_folder/capistrano'
|
7
|
+
#
|
8
|
+
Capistrano::Configuration.instance.load do
|
9
|
+
after 'deploy:stop', 'vidibus:watch_folder:stop'
|
10
|
+
after 'deploy:start', 'vidibus:watch_folder:start'
|
11
|
+
after 'deploy:restart', 'vidibus:watch_folder:restart'
|
12
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
begin
|
2
|
+
require 'daemons'
|
3
|
+
rescue LoadError
|
4
|
+
raise %(Please add `gem 'daemons' gem to your Gemfile for this to work)
|
5
|
+
end
|
6
|
+
require 'optparse'
|
7
|
+
|
8
|
+
module Vidibus
|
9
|
+
module WatchFolder
|
10
|
+
class Daemon
|
11
|
+
|
12
|
+
def initialize(args)
|
13
|
+
@options = {:pid_dir => "#{Rails.root}/tmp/pids"}
|
14
|
+
options = OptionParser.new do |options|
|
15
|
+
options.banner = "Usage: #{File.basename($0)} start|stop|restart"
|
16
|
+
options.on('-h', '--help', 'Show this message') do
|
17
|
+
puts options
|
18
|
+
exit 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
@args = options.parse!(args)
|
22
|
+
end
|
23
|
+
|
24
|
+
def daemonize
|
25
|
+
dir = @options[:pid_dir]
|
26
|
+
Dir.mkdir(dir) unless File.exists?(dir)
|
27
|
+
run_process('watch_folder', dir)
|
28
|
+
end
|
29
|
+
|
30
|
+
def run_process(name, dir)
|
31
|
+
Daemons.run_proc(name, :dir => dir, :dir_mode => :normal) { run }
|
32
|
+
end
|
33
|
+
|
34
|
+
def run
|
35
|
+
Dir.chdir(Rails.root)
|
36
|
+
log = File.join(Rails.root, 'log', 'watch_folder.log')
|
37
|
+
Vidibus::WatchFolder.logger = ActiveSupport::BufferedLogger.new(log)
|
38
|
+
Vidibus::WatchFolder.listen
|
39
|
+
rescue => e
|
40
|
+
Vidibus::WatchFolder.logger.fatal(e)
|
41
|
+
STDERR.puts(e.message)
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'delayed_job_mongoid'
|
2
|
+
|
3
|
+
module Vidibus
|
4
|
+
module WatchFolder
|
5
|
+
class Job < Struct.new(:uuid, :event, :path, :checksum, :delay)
|
6
|
+
def enqueue!
|
7
|
+
validate!
|
8
|
+
args = [self]
|
9
|
+
i = delay.to_i
|
10
|
+
if i > 0
|
11
|
+
args << {:run_at => Time.now+i}
|
12
|
+
end
|
13
|
+
Delayed::Job.enqueue(*args).id
|
14
|
+
end
|
15
|
+
|
16
|
+
def perform
|
17
|
+
begin
|
18
|
+
watch_folder.handle(event, path, checksum)
|
19
|
+
rescue Mongoid::Errors::DocumentNotFound
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def validate!
|
24
|
+
return if uuid && event && path && checksum
|
25
|
+
raise ArgumentError, 'Provide UUID, event, path, checksum, and an optional delay'
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.create(*args)
|
29
|
+
new(*args).enqueue!
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def watch_folder
|
35
|
+
Base.find_by_uuid(uuid)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Vidibus
|
2
|
+
module WatchFolder
|
3
|
+
module Util
|
4
|
+
module Directory
|
5
|
+
extend self
|
6
|
+
|
7
|
+
# Check if path is a read- and writable directory.
|
8
|
+
def valid?(path)
|
9
|
+
File.exist?(path) &&
|
10
|
+
File.directory?(path) &&
|
11
|
+
File.readable?(path) &&
|
12
|
+
File.writable?(path)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'vidibus/watch_folder/util/directory'
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'vidibus/watch_folder/util'
|
2
|
+
require 'vidibus/watch_folder/base'
|
3
|
+
require 'vidibus/watch_folder/job'
|
4
|
+
require 'vidibus/watch_folder/railtie' if defined?(Rails::Railtie)
|
5
|
+
|
6
|
+
require 'listen'
|
7
|
+
|
8
|
+
module Vidibus
|
9
|
+
module WatchFolder
|
10
|
+
extend self
|
11
|
+
|
12
|
+
class Error < StandardError; end
|
13
|
+
class NoRootsError < Error; end
|
14
|
+
|
15
|
+
EVENTS = %w[added modified removed]
|
16
|
+
|
17
|
+
attr_accessor :roots, :logger, :autoload_paths, :path_mapping
|
18
|
+
@roots = []
|
19
|
+
@logger = Logger.new(STDOUT)
|
20
|
+
@autoload_paths = []
|
21
|
+
@path_mapping = []
|
22
|
+
|
23
|
+
# Calculate checksum of given file path
|
24
|
+
def checksum(path)
|
25
|
+
Digest::SHA2.file(path).hexdigest
|
26
|
+
end
|
27
|
+
|
28
|
+
# Listen for changes within all roots
|
29
|
+
def listen
|
30
|
+
autoload
|
31
|
+
unless roots.any?
|
32
|
+
raise NoRootsError, 'No folders to watch!'
|
33
|
+
end
|
34
|
+
roots.uniq!
|
35
|
+
logger.debug("[#{Time.now.utc}] - Listening to #{roots.join(',')}")
|
36
|
+
args = roots + [{:latency => 0.1}]
|
37
|
+
Listen.to(*args) do |modified, added, removed|
|
38
|
+
EVENTS.each do |event|
|
39
|
+
eval(event).each do |path|
|
40
|
+
logger.debug %([#{Time.now.utc}] - #{event}: #{path})
|
41
|
+
begin
|
42
|
+
uuid = path[/^#{roots_regex}\/([^\/]+)\/.+$/, 1] || next
|
43
|
+
begin
|
44
|
+
base = Base.find_by_uuid(uuid)
|
45
|
+
base.handle(event, path)
|
46
|
+
rescue Mongoid::Errors::DocumentNotFound
|
47
|
+
logger.error %([#{Time.now.utc}] - Can't find Vidibus::WatchFolder::Base #{uuid})
|
48
|
+
end
|
49
|
+
rescue => e
|
50
|
+
logger.error("[#{Time.now.utc}] - ERROR in Vidibus::WatchFolder.listen:\n#{e.inspect}\n---\n#{e.backtrace.join("\n")}")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Constantize all watch folder class names to trigger autoloading.
|
58
|
+
def autoload
|
59
|
+
return unless autoload_paths.any?
|
60
|
+
list = Dir[*autoload_paths].map do |f|
|
61
|
+
File.read(f)[/class ([^<]+) < Vidibus::WatchFolder::Base/, 1]
|
62
|
+
end.compact.map { |k| k.constantize }
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
# Return regular expression for root paths.
|
68
|
+
#
|
69
|
+
# If any path_mapping has been defined, that mapping will be applied.
|
70
|
+
# That is often required to turn absolute path into relative ones in order
|
71
|
+
# to avoid problems with symlinks, because uploaded files will usually go
|
72
|
+
# into a shared directory but the root paths Rails reports are from within
|
73
|
+
# the current release directory.
|
74
|
+
def roots_regex
|
75
|
+
@roots_regex ||= begin
|
76
|
+
_roots = roots.join('|')
|
77
|
+
path_mapping.each do |from, to|
|
78
|
+
_roots.gsub!(from, to)
|
79
|
+
end
|
80
|
+
/(?:#{_roots})/
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'vidibus/watch_folder'
|
metadata
ADDED
@@ -0,0 +1,256 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: vidibus-watch_folder
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- André Pankratz
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-28 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: mongoid
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.5'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '2.5'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: listen
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0.5'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0.5'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rb-fsevent
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.9.1
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.9.1
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: rb-inotify
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.8.8
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.8.8
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: vidibus-uuid
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: delayed_job_mongoid
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: bundler
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 1.0.0
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: 1.0.0
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: rake
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: rdoc
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
- !ruby/object:Gem::Dependency
|
159
|
+
name: simplecov
|
160
|
+
requirement: !ruby/object:Gem::Requirement
|
161
|
+
none: false
|
162
|
+
requirements:
|
163
|
+
- - ! '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
none: false
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: rspec
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ! '>='
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
- !ruby/object:Gem::Dependency
|
191
|
+
name: rr
|
192
|
+
requirement: !ruby/object:Gem::Requirement
|
193
|
+
none: false
|
194
|
+
requirements:
|
195
|
+
- - ! '>='
|
196
|
+
- !ruby/object:Gem::Version
|
197
|
+
version: '0'
|
198
|
+
type: :development
|
199
|
+
prerelease: false
|
200
|
+
version_requirements: !ruby/object:Gem::Requirement
|
201
|
+
none: false
|
202
|
+
requirements:
|
203
|
+
- - ! '>='
|
204
|
+
- !ruby/object:Gem::Version
|
205
|
+
version: '0'
|
206
|
+
description: Create multiple watch folders within your application, e.g. to provide
|
207
|
+
individual FTP mount points for customers.
|
208
|
+
email: andre@vidibus.com
|
209
|
+
executables: []
|
210
|
+
extensions: []
|
211
|
+
extra_rdoc_files: []
|
212
|
+
files:
|
213
|
+
- lib/generators/vidibus/templates/script
|
214
|
+
- lib/generators/vidibus/watch_folder_generator.rb
|
215
|
+
- lib/vidibus/watch_folder/base.rb
|
216
|
+
- lib/vidibus/watch_folder/capistrano/recipes.rb
|
217
|
+
- lib/vidibus/watch_folder/capistrano.rb
|
218
|
+
- lib/vidibus/watch_folder/daemon.rb
|
219
|
+
- lib/vidibus/watch_folder/job.rb
|
220
|
+
- lib/vidibus/watch_folder/railtie.rb
|
221
|
+
- lib/vidibus/watch_folder/util/directory.rb
|
222
|
+
- lib/vidibus/watch_folder/util.rb
|
223
|
+
- lib/vidibus/watch_folder/version.rb
|
224
|
+
- lib/vidibus/watch_folder.rb
|
225
|
+
- lib/vidibus-watch_folder.rb
|
226
|
+
- LICENSE
|
227
|
+
- README.md
|
228
|
+
- Rakefile
|
229
|
+
homepage: https://github.com/vidibus/vidibus-watch_folder
|
230
|
+
licenses: []
|
231
|
+
post_install_message:
|
232
|
+
rdoc_options: []
|
233
|
+
require_paths:
|
234
|
+
- lib
|
235
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
236
|
+
none: false
|
237
|
+
requirements:
|
238
|
+
- - ! '>='
|
239
|
+
- !ruby/object:Gem::Version
|
240
|
+
version: '0'
|
241
|
+
segments:
|
242
|
+
- 0
|
243
|
+
hash: -1021260411944907488
|
244
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
245
|
+
none: false
|
246
|
+
requirements:
|
247
|
+
- - ! '>='
|
248
|
+
- !ruby/object:Gem::Version
|
249
|
+
version: 1.3.6
|
250
|
+
requirements: []
|
251
|
+
rubyforge_project: vidibus-watch_folder
|
252
|
+
rubygems_version: 1.8.24
|
253
|
+
signing_key:
|
254
|
+
specification_version: 3
|
255
|
+
summary: Watch folders based on Mongoid with asynchronous processing
|
256
|
+
test_files: []
|