rivendell-import 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +21 -0
- data/Gemfile +11 -0
- data/Guardfile +14 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +4 -0
- data/bin/rivendell-import +6 -0
- data/db/migrate/20120920712200_create_tasks.rb +21 -0
- data/db/migrate/20120927194300_create_notifiers.rb +14 -0
- data/db/migrate/20120927203600_create_notifications.rb +14 -0
- data/examples/.gitignore +2 -0
- data/examples/config.rb +31 -0
- data/features/manage_cart_attributes.feature +20 -0
- data/features/manage_file_matching.feature +44 -0
- data/features/step_definitions/import_steps.rb +38 -0
- data/features/support/env.rb +19 -0
- data/features/support/mock_xport.rb +34 -0
- data/lib/rivendell/import.rb +48 -0
- data/lib/rivendell/import/base.rb +57 -0
- data/lib/rivendell/import/cart.rb +67 -0
- data/lib/rivendell/import/carts_cache.rb +58 -0
- data/lib/rivendell/import/cli.rb +90 -0
- data/lib/rivendell/import/config.rb +13 -0
- data/lib/rivendell/import/context.rb +28 -0
- data/lib/rivendell/import/cut.rb +33 -0
- data/lib/rivendell/import/file.rb +57 -0
- data/lib/rivendell/import/notification.rb +16 -0
- data/lib/rivendell/import/notifier/base.rb +89 -0
- data/lib/rivendell/import/notifier/mail-body.erb +8 -0
- data/lib/rivendell/import/notifier/mail-subject.erb +11 -0
- data/lib/rivendell/import/notifier/mail.rb +104 -0
- data/lib/rivendell/import/notifiers.rb +24 -0
- data/lib/rivendell/import/task.rb +80 -0
- data/lib/rivendell/import/tasking/cart.rb +25 -0
- data/lib/rivendell/import/tasking/destination.rb +28 -0
- data/lib/rivendell/import/tasking/file.rb +20 -0
- data/lib/rivendell/import/tasking/status.rb +29 -0
- data/lib/rivendell/import/tasking/tags.rb +27 -0
- data/lib/rivendell/import/tasks.rb +23 -0
- data/lib/rivendell/import/version.rb +5 -0
- data/lib/rivendell/import/worker.rb +34 -0
- data/rivendell-import.gemspec +37 -0
- data/spec/fixtures/mail-body.erb +5 -0
- data/spec/rivendell/import/base_spec.rb +125 -0
- data/spec/rivendell/import/cart_spec.rb +132 -0
- data/spec/rivendell/import/carts_cache_spec.rb +110 -0
- data/spec/rivendell/import/cli_spec.rb +149 -0
- data/spec/rivendell/import/config_spec.rb +16 -0
- data/spec/rivendell/import/context_spec.rb +19 -0
- data/spec/rivendell/import/file_spec.rb +63 -0
- data/spec/rivendell/import/notifier/base_spec.rb +63 -0
- data/spec/rivendell/import/notifier/mail_spec.rb +110 -0
- data/spec/rivendell/import/task_spec.rb +217 -0
- data/spec/rivendell/import/tasks_spec.rb +30 -0
- data/spec/rivendell/import/worker_spec.rb +25 -0
- data/spec/rivendell/import_spec.rb +32 -0
- data/spec/spec_helper.rb +15 -0
- data/spec/support/database_cleaner.rb +17 -0
- data/spec/support/fixtures.rb +3 -0
- data/spec/support/mail.rb +3 -0
- data/spec/support/test_notifier.rb +15 -0
- data/tasks/ci.rake +2 -0
- data/tasks/cucumber.rake +4 -0
- data/tasks/database.rake +11 -0
- data/tasks/rdoc.rake +16 -0
- data/tasks/rspec.rake +2 -0
- metadata +399 -0
@@ -0,0 +1,67 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Cart
|
3
|
+
|
4
|
+
include ActiveModel::Serialization
|
5
|
+
include ActiveModel::Serializers::JSON
|
6
|
+
|
7
|
+
def attributes
|
8
|
+
%w{number group}.inject({}) do |map, attribute|
|
9
|
+
value = send attribute
|
10
|
+
map[attribute] = value if value
|
11
|
+
map
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes=(attributes)
|
16
|
+
attributes.each { |k,v| send "#{k}=", v }
|
17
|
+
end
|
18
|
+
|
19
|
+
delegate :blank?, :to => :attributes
|
20
|
+
|
21
|
+
attr_accessor :number, :group
|
22
|
+
attr_reader :task
|
23
|
+
|
24
|
+
def initialize(task = nil)
|
25
|
+
@task = task
|
26
|
+
end
|
27
|
+
|
28
|
+
def xport
|
29
|
+
task.xport
|
30
|
+
end
|
31
|
+
|
32
|
+
def create
|
33
|
+
unless number
|
34
|
+
raise "Can't create Cart, Group isn't defined" unless group.present?
|
35
|
+
self.number = xport.add_cart(:group => group).number
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def update
|
40
|
+
# xport.edit_cart # to define title, etc ...
|
41
|
+
end
|
42
|
+
|
43
|
+
def cut
|
44
|
+
@cut ||= Cut.new(self)
|
45
|
+
end
|
46
|
+
|
47
|
+
def import(file)
|
48
|
+
raise "File #{file.path} not found" unless file.exists?
|
49
|
+
|
50
|
+
cut.create
|
51
|
+
xport.import number, cut.number, file.path
|
52
|
+
cut.update
|
53
|
+
end
|
54
|
+
|
55
|
+
def find_by_title(string, options = {})
|
56
|
+
if remote_cart = carts_cache.find_by_title(string, options)
|
57
|
+
self.number = remote_cart.number
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def carts_cache
|
62
|
+
@carts_cache ||= Rivendell::Import::CartsCache.new(xport)
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class CartsCache
|
3
|
+
|
4
|
+
attr_accessor :xport
|
5
|
+
|
6
|
+
def initialize(xport)
|
7
|
+
@xport = xport
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_all_by_title(string, options = {})
|
11
|
+
normalizer = options.delete(:normalizer) || Proc.new { |id| id }
|
12
|
+
|
13
|
+
string = normalizer.call(string)
|
14
|
+
carts(options).select do |cart|
|
15
|
+
normalizer.call(cart.title) == string
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def default_normalizer
|
20
|
+
Proc.new do |string|
|
21
|
+
string.downcase.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def find_by_title(string, options = {})
|
26
|
+
matching_carts = find_all_by_title(string, options)
|
27
|
+
|
28
|
+
if matching_carts.blank?
|
29
|
+
matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
|
30
|
+
end
|
31
|
+
|
32
|
+
matching_carts.first if matching_carts.one?
|
33
|
+
end
|
34
|
+
|
35
|
+
def cache
|
36
|
+
clear if purged_at < time_to_live.ago
|
37
|
+
@cache ||= {}
|
38
|
+
end
|
39
|
+
|
40
|
+
def clear
|
41
|
+
@cache = nil
|
42
|
+
end
|
43
|
+
|
44
|
+
@@time_to_live = 600
|
45
|
+
cattr_accessor :time_to_live
|
46
|
+
|
47
|
+
attr_accessor :purged_at
|
48
|
+
|
49
|
+
def purged_at
|
50
|
+
@purged_at ||= Time.now
|
51
|
+
end
|
52
|
+
|
53
|
+
def carts(options = {})
|
54
|
+
cache[options.to_s] ||= xport.list_carts(options)
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'trollop'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Rivendell::Import
|
5
|
+
class CLI
|
6
|
+
|
7
|
+
attr_reader :arguments
|
8
|
+
attr_accessor :options_count
|
9
|
+
|
10
|
+
def initialize(arguments = [])
|
11
|
+
@arguments = arguments
|
12
|
+
end
|
13
|
+
|
14
|
+
def config_file
|
15
|
+
@config_file ||= options[:config]
|
16
|
+
end
|
17
|
+
|
18
|
+
def listen_mode?
|
19
|
+
options[:listen]
|
20
|
+
end
|
21
|
+
|
22
|
+
def dry_run?
|
23
|
+
options[:dry_run]
|
24
|
+
end
|
25
|
+
|
26
|
+
def debug?
|
27
|
+
options[:debug]
|
28
|
+
end
|
29
|
+
|
30
|
+
def database
|
31
|
+
options[:database]
|
32
|
+
end
|
33
|
+
|
34
|
+
def parser
|
35
|
+
@parser ||= Trollop::Parser.new do
|
36
|
+
opt :config, "Configuration file", :type => String
|
37
|
+
opt :listen, "Wait for files in given directory"
|
38
|
+
opt :dry_run, "Just create tasks without executing them"
|
39
|
+
opt :debug, "Enable debug messages (in stderr)"
|
40
|
+
opt :database, "The database file used to store tasks", :type => String
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def options
|
45
|
+
@options ||= Trollop::with_standard_exception_handling(parser) do
|
46
|
+
raise Trollop::HelpNeeded if ARGV.empty? # show help screen
|
47
|
+
parser.parse arguments
|
48
|
+
end
|
49
|
+
end
|
50
|
+
alias_method :parse, :options
|
51
|
+
|
52
|
+
def parsed_parser
|
53
|
+
parse
|
54
|
+
parser
|
55
|
+
end
|
56
|
+
|
57
|
+
def import
|
58
|
+
@import ||= Rivendell::Import::Base.new
|
59
|
+
end
|
60
|
+
|
61
|
+
def paths
|
62
|
+
parsed_parser.leftovers
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
66
|
+
Rivendell::Import.logger = Logger.new($stderr) if debug?
|
67
|
+
|
68
|
+
if database
|
69
|
+
Rivendell::Import.establish_connection database
|
70
|
+
else
|
71
|
+
Rivendell::Import.establish_connection
|
72
|
+
end
|
73
|
+
|
74
|
+
if config_file
|
75
|
+
load config_file
|
76
|
+
import.to_prepare = Rivendell::Import.config.to_prepare
|
77
|
+
end
|
78
|
+
|
79
|
+
if listen_mode?
|
80
|
+
listen_options = {}
|
81
|
+
listen_options[:dry_run] = true if dry_run?
|
82
|
+
|
83
|
+
import.listen paths.first, listen_options
|
84
|
+
else
|
85
|
+
import.process paths
|
86
|
+
import.tasks.run unless dry_run?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Context
|
3
|
+
|
4
|
+
attr_reader :task
|
5
|
+
|
6
|
+
def initialize(task)
|
7
|
+
@task = task
|
8
|
+
end
|
9
|
+
|
10
|
+
delegate :file, :cart, :logger, :to => :task
|
11
|
+
|
12
|
+
def with(expression)
|
13
|
+
yield if file.match expression
|
14
|
+
end
|
15
|
+
|
16
|
+
def notify(target, options = {})
|
17
|
+
Rivendell::Import::Notifier::Base.notify(target, options).tap do |notifier|
|
18
|
+
logger.debug "Will notify with #{notifier.inspect}"
|
19
|
+
task.notifiers << notifier
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def run(&block)
|
24
|
+
instance_exec file, &block if block_given?
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Cut
|
3
|
+
|
4
|
+
include ActiveModel::Serialization
|
5
|
+
include ActiveModel::Serializers::JSON
|
6
|
+
|
7
|
+
attr_reader :cart
|
8
|
+
|
9
|
+
attr_accessor :number
|
10
|
+
|
11
|
+
def initialize(cart)
|
12
|
+
@cart = cart
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes
|
16
|
+
{}
|
17
|
+
end
|
18
|
+
|
19
|
+
def xport
|
20
|
+
cart.xport
|
21
|
+
end
|
22
|
+
|
23
|
+
def create
|
24
|
+
# xport.delete_cuts # if clean cuts is required
|
25
|
+
self.number = xport.add_cut(cart.number).number unless number
|
26
|
+
end
|
27
|
+
|
28
|
+
def update
|
29
|
+
# xport.edit_cut # to define attributes
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class File
|
3
|
+
|
4
|
+
attr_reader :name, :path
|
5
|
+
|
6
|
+
def initialize(name, attributes = {})
|
7
|
+
if attributes[:base_directory]
|
8
|
+
@path = name
|
9
|
+
@name = self.class.relative_filename(name, attributes[:base_directory])
|
10
|
+
elsif attributes[:path]
|
11
|
+
@name = name
|
12
|
+
@path = attributes[:path]
|
13
|
+
else
|
14
|
+
@name = @path = name
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.relative_filename(path, base_directory)
|
19
|
+
::File.expand_path(path).gsub(%r{^#{::File.expand_path(base_directory)}/},"")
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_s
|
23
|
+
name
|
24
|
+
end
|
25
|
+
|
26
|
+
def basename
|
27
|
+
::File.basename(name, ".#{extension}")
|
28
|
+
end
|
29
|
+
|
30
|
+
def extension
|
31
|
+
::File.extname(name).gsub(/^\./,'')
|
32
|
+
end
|
33
|
+
|
34
|
+
def ==(other)
|
35
|
+
other and path == other.path
|
36
|
+
end
|
37
|
+
|
38
|
+
def match(expression)
|
39
|
+
name.match(expression).tap do |result|
|
40
|
+
verb = result ? "match" : "doesn't match"
|
41
|
+
Rivendell::Import.logger.debug "File #{verb} '#{expression}'"
|
42
|
+
!!result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def in(directory, &block)
|
47
|
+
if match %r{^#{directory}/}
|
48
|
+
yield
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def exists?
|
53
|
+
::File.exists? path
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Notification < ActiveRecord::Base
|
3
|
+
|
4
|
+
belongs_to :task
|
5
|
+
belongs_to :notifier, :class_name => "Rivendell::Import::Notifier::Base"
|
6
|
+
|
7
|
+
def sent?
|
8
|
+
sent_at.present?
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.pending
|
12
|
+
where :sent_at => nil
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'mail'
|
2
|
+
require 'erb'
|
3
|
+
|
4
|
+
module Rivendell::Import::Notifier
|
5
|
+
class Base < ActiveRecord::Base
|
6
|
+
|
7
|
+
# self.abstract_class = true
|
8
|
+
self.table_name = "notifiers"
|
9
|
+
|
10
|
+
def parameters
|
11
|
+
{}
|
12
|
+
end
|
13
|
+
|
14
|
+
has_many :notifications, :foreign_key => "notifier_id" do
|
15
|
+
def to_sent
|
16
|
+
pending.joins(:task).where("tasks.status" => %w{completed failed})
|
17
|
+
end
|
18
|
+
def waiting_for_more_than(delay)
|
19
|
+
joins(:task).where("tasks.updated_at < ?", Time.now - delay)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
has_many :tasks, :through => :notifications
|
24
|
+
|
25
|
+
def delay
|
26
|
+
tasks.pending.empty? ? 0 : 60
|
27
|
+
end
|
28
|
+
|
29
|
+
def notify
|
30
|
+
to_sent_notifications = notifications.to_sent.includes(:task)
|
31
|
+
|
32
|
+
if delay > 0
|
33
|
+
to_sent_notifications = to_sent_notifications.waiting_for_more_than(delay)
|
34
|
+
end
|
35
|
+
|
36
|
+
if to_sent_notifications.present?
|
37
|
+
logger.debug "Notify #{to_sent_notifications.size} tasks with #{self.inspect}"
|
38
|
+
notify! to_sent_notifications.map(&:task)
|
39
|
+
to_sent_notifications.update_all(:sent_at => Time.now)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def logger
|
44
|
+
Rivendell::Import.logger
|
45
|
+
end
|
46
|
+
|
47
|
+
def parameters=(parameters)
|
48
|
+
parameters.each { |k,v| send "#{k}=", v }
|
49
|
+
end
|
50
|
+
|
51
|
+
def raw_parameters
|
52
|
+
read_attribute :parameters
|
53
|
+
end
|
54
|
+
|
55
|
+
def read_parameters
|
56
|
+
self.parameters = ActiveSupport::JSON.decode(raw_parameters) if raw_parameters.present?
|
57
|
+
end
|
58
|
+
|
59
|
+
after_initialize :read_parameters
|
60
|
+
|
61
|
+
def write_parameters
|
62
|
+
if parameters.present?
|
63
|
+
write_attribute :parameters, parameters.to_json
|
64
|
+
write_attribute :key, parameters.hash
|
65
|
+
else
|
66
|
+
write_attribute :parameters, nil
|
67
|
+
write_attribute :key, nil
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
before_save :write_parameters
|
72
|
+
|
73
|
+
def self.notify(target, options = {})
|
74
|
+
new_notifier = case options.delete(:by)
|
75
|
+
when :email
|
76
|
+
Mail.new options.merge(:to => target)
|
77
|
+
end
|
78
|
+
|
79
|
+
key = new_notifier.parameters.hash
|
80
|
+
if existing_notifier = where(:type => new_notifier.type, :key => key).first
|
81
|
+
existing_notifier
|
82
|
+
else
|
83
|
+
new_notifier.save!
|
84
|
+
new_notifier
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|