rivendell-import 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +21 -0
- data/Gemfile +11 -0
- data/Guardfile +14 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +4 -0
- data/bin/rivendell-import +6 -0
- data/db/migrate/20120920712200_create_tasks.rb +21 -0
- data/db/migrate/20120927194300_create_notifiers.rb +14 -0
- data/db/migrate/20120927203600_create_notifications.rb +14 -0
- data/examples/.gitignore +2 -0
- data/examples/config.rb +31 -0
- data/features/manage_cart_attributes.feature +20 -0
- data/features/manage_file_matching.feature +44 -0
- data/features/step_definitions/import_steps.rb +38 -0
- data/features/support/env.rb +19 -0
- data/features/support/mock_xport.rb +34 -0
- data/lib/rivendell/import.rb +48 -0
- data/lib/rivendell/import/base.rb +57 -0
- data/lib/rivendell/import/cart.rb +67 -0
- data/lib/rivendell/import/carts_cache.rb +58 -0
- data/lib/rivendell/import/cli.rb +90 -0
- data/lib/rivendell/import/config.rb +13 -0
- data/lib/rivendell/import/context.rb +28 -0
- data/lib/rivendell/import/cut.rb +33 -0
- data/lib/rivendell/import/file.rb +57 -0
- data/lib/rivendell/import/notification.rb +16 -0
- data/lib/rivendell/import/notifier/base.rb +89 -0
- data/lib/rivendell/import/notifier/mail-body.erb +8 -0
- data/lib/rivendell/import/notifier/mail-subject.erb +11 -0
- data/lib/rivendell/import/notifier/mail.rb +104 -0
- data/lib/rivendell/import/notifiers.rb +24 -0
- data/lib/rivendell/import/task.rb +80 -0
- data/lib/rivendell/import/tasking/cart.rb +25 -0
- data/lib/rivendell/import/tasking/destination.rb +28 -0
- data/lib/rivendell/import/tasking/file.rb +20 -0
- data/lib/rivendell/import/tasking/status.rb +29 -0
- data/lib/rivendell/import/tasking/tags.rb +27 -0
- data/lib/rivendell/import/tasks.rb +23 -0
- data/lib/rivendell/import/version.rb +5 -0
- data/lib/rivendell/import/worker.rb +34 -0
- data/rivendell-import.gemspec +37 -0
- data/spec/fixtures/mail-body.erb +5 -0
- data/spec/rivendell/import/base_spec.rb +125 -0
- data/spec/rivendell/import/cart_spec.rb +132 -0
- data/spec/rivendell/import/carts_cache_spec.rb +110 -0
- data/spec/rivendell/import/cli_spec.rb +149 -0
- data/spec/rivendell/import/config_spec.rb +16 -0
- data/spec/rivendell/import/context_spec.rb +19 -0
- data/spec/rivendell/import/file_spec.rb +63 -0
- data/spec/rivendell/import/notifier/base_spec.rb +63 -0
- data/spec/rivendell/import/notifier/mail_spec.rb +110 -0
- data/spec/rivendell/import/task_spec.rb +217 -0
- data/spec/rivendell/import/tasks_spec.rb +30 -0
- data/spec/rivendell/import/worker_spec.rb +25 -0
- data/spec/rivendell/import_spec.rb +32 -0
- data/spec/spec_helper.rb +15 -0
- data/spec/support/database_cleaner.rb +17 -0
- data/spec/support/fixtures.rb +3 -0
- data/spec/support/mail.rb +3 -0
- data/spec/support/test_notifier.rb +15 -0
- data/tasks/ci.rake +2 -0
- data/tasks/cucumber.rake +4 -0
- data/tasks/database.rake +11 -0
- data/tasks/rdoc.rake +16 -0
- data/tasks/rspec.rake +2 -0
- metadata +399 -0
@@ -0,0 +1,67 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Cart
|
3
|
+
|
4
|
+
include ActiveModel::Serialization
|
5
|
+
include ActiveModel::Serializers::JSON
|
6
|
+
|
7
|
+
def attributes
|
8
|
+
%w{number group}.inject({}) do |map, attribute|
|
9
|
+
value = send attribute
|
10
|
+
map[attribute] = value if value
|
11
|
+
map
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes=(attributes)
|
16
|
+
attributes.each { |k,v| send "#{k}=", v }
|
17
|
+
end
|
18
|
+
|
19
|
+
delegate :blank?, :to => :attributes
|
20
|
+
|
21
|
+
attr_accessor :number, :group
|
22
|
+
attr_reader :task
|
23
|
+
|
24
|
+
def initialize(task = nil)
|
25
|
+
@task = task
|
26
|
+
end
|
27
|
+
|
28
|
+
def xport
|
29
|
+
task.xport
|
30
|
+
end
|
31
|
+
|
32
|
+
def create
|
33
|
+
unless number
|
34
|
+
raise "Can't create Cart, Group isn't defined" unless group.present?
|
35
|
+
self.number = xport.add_cart(:group => group).number
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def update
|
40
|
+
# xport.edit_cart # to define title, etc ...
|
41
|
+
end
|
42
|
+
|
43
|
+
def cut
|
44
|
+
@cut ||= Cut.new(self)
|
45
|
+
end
|
46
|
+
|
47
|
+
def import(file)
|
48
|
+
raise "File #{file.path} not found" unless file.exists?
|
49
|
+
|
50
|
+
cut.create
|
51
|
+
xport.import number, cut.number, file.path
|
52
|
+
cut.update
|
53
|
+
end
|
54
|
+
|
55
|
+
def find_by_title(string, options = {})
|
56
|
+
if remote_cart = carts_cache.find_by_title(string, options)
|
57
|
+
self.number = remote_cart.number
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def carts_cache
|
62
|
+
@carts_cache ||= Rivendell::Import::CartsCache.new(xport)
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class CartsCache
|
3
|
+
|
4
|
+
attr_accessor :xport
|
5
|
+
|
6
|
+
def initialize(xport)
|
7
|
+
@xport = xport
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_all_by_title(string, options = {})
|
11
|
+
normalizer = options.delete(:normalizer) || Proc.new { |id| id }
|
12
|
+
|
13
|
+
string = normalizer.call(string)
|
14
|
+
carts(options).select do |cart|
|
15
|
+
normalizer.call(cart.title) == string
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def default_normalizer
|
20
|
+
Proc.new do |string|
|
21
|
+
string.downcase.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def find_by_title(string, options = {})
|
26
|
+
matching_carts = find_all_by_title(string, options)
|
27
|
+
|
28
|
+
if matching_carts.blank?
|
29
|
+
matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
|
30
|
+
end
|
31
|
+
|
32
|
+
matching_carts.first if matching_carts.one?
|
33
|
+
end
|
34
|
+
|
35
|
+
def cache
|
36
|
+
clear if purged_at < time_to_live.ago
|
37
|
+
@cache ||= {}
|
38
|
+
end
|
39
|
+
|
40
|
+
def clear
|
41
|
+
@cache = nil
|
42
|
+
end
|
43
|
+
|
44
|
+
@@time_to_live = 600
|
45
|
+
cattr_accessor :time_to_live
|
46
|
+
|
47
|
+
attr_accessor :purged_at
|
48
|
+
|
49
|
+
def purged_at
|
50
|
+
@purged_at ||= Time.now
|
51
|
+
end
|
52
|
+
|
53
|
+
def carts(options = {})
|
54
|
+
cache[options.to_s] ||= xport.list_carts(options)
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'trollop'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Rivendell::Import
|
5
|
+
class CLI
|
6
|
+
|
7
|
+
attr_reader :arguments
|
8
|
+
attr_accessor :options_count
|
9
|
+
|
10
|
+
def initialize(arguments = [])
|
11
|
+
@arguments = arguments
|
12
|
+
end
|
13
|
+
|
14
|
+
def config_file
|
15
|
+
@config_file ||= options[:config]
|
16
|
+
end
|
17
|
+
|
18
|
+
def listen_mode?
|
19
|
+
options[:listen]
|
20
|
+
end
|
21
|
+
|
22
|
+
def dry_run?
|
23
|
+
options[:dry_run]
|
24
|
+
end
|
25
|
+
|
26
|
+
def debug?
|
27
|
+
options[:debug]
|
28
|
+
end
|
29
|
+
|
30
|
+
def database
|
31
|
+
options[:database]
|
32
|
+
end
|
33
|
+
|
34
|
+
def parser
|
35
|
+
@parser ||= Trollop::Parser.new do
|
36
|
+
opt :config, "Configuration file", :type => String
|
37
|
+
opt :listen, "Wait for files in given directory"
|
38
|
+
opt :dry_run, "Just create tasks without executing them"
|
39
|
+
opt :debug, "Enable debug messages (in stderr)"
|
40
|
+
opt :database, "The database file used to store tasks", :type => String
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def options
|
45
|
+
@options ||= Trollop::with_standard_exception_handling(parser) do
|
46
|
+
raise Trollop::HelpNeeded if ARGV.empty? # show help screen
|
47
|
+
parser.parse arguments
|
48
|
+
end
|
49
|
+
end
|
50
|
+
alias_method :parse, :options
|
51
|
+
|
52
|
+
def parsed_parser
|
53
|
+
parse
|
54
|
+
parser
|
55
|
+
end
|
56
|
+
|
57
|
+
def import
|
58
|
+
@import ||= Rivendell::Import::Base.new
|
59
|
+
end
|
60
|
+
|
61
|
+
def paths
|
62
|
+
parsed_parser.leftovers
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
66
|
+
Rivendell::Import.logger = Logger.new($stderr) if debug?
|
67
|
+
|
68
|
+
if database
|
69
|
+
Rivendell::Import.establish_connection database
|
70
|
+
else
|
71
|
+
Rivendell::Import.establish_connection
|
72
|
+
end
|
73
|
+
|
74
|
+
if config_file
|
75
|
+
load config_file
|
76
|
+
import.to_prepare = Rivendell::Import.config.to_prepare
|
77
|
+
end
|
78
|
+
|
79
|
+
if listen_mode?
|
80
|
+
listen_options = {}
|
81
|
+
listen_options[:dry_run] = true if dry_run?
|
82
|
+
|
83
|
+
import.listen paths.first, listen_options
|
84
|
+
else
|
85
|
+
import.process paths
|
86
|
+
import.tasks.run unless dry_run?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Context
|
3
|
+
|
4
|
+
attr_reader :task
|
5
|
+
|
6
|
+
def initialize(task)
|
7
|
+
@task = task
|
8
|
+
end
|
9
|
+
|
10
|
+
delegate :file, :cart, :logger, :to => :task
|
11
|
+
|
12
|
+
def with(expression)
|
13
|
+
yield if file.match expression
|
14
|
+
end
|
15
|
+
|
16
|
+
def notify(target, options = {})
|
17
|
+
Rivendell::Import::Notifier::Base.notify(target, options).tap do |notifier|
|
18
|
+
logger.debug "Will notify with #{notifier.inspect}"
|
19
|
+
task.notifiers << notifier
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def run(&block)
|
24
|
+
instance_exec file, &block if block_given?
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Cut
|
3
|
+
|
4
|
+
include ActiveModel::Serialization
|
5
|
+
include ActiveModel::Serializers::JSON
|
6
|
+
|
7
|
+
attr_reader :cart
|
8
|
+
|
9
|
+
attr_accessor :number
|
10
|
+
|
11
|
+
def initialize(cart)
|
12
|
+
@cart = cart
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes
|
16
|
+
{}
|
17
|
+
end
|
18
|
+
|
19
|
+
def xport
|
20
|
+
cart.xport
|
21
|
+
end
|
22
|
+
|
23
|
+
def create
|
24
|
+
# xport.delete_cuts # if clean cuts is required
|
25
|
+
self.number = xport.add_cut(cart.number).number unless number
|
26
|
+
end
|
27
|
+
|
28
|
+
def update
|
29
|
+
# xport.edit_cut # to define attributes
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class File
|
3
|
+
|
4
|
+
attr_reader :name, :path
|
5
|
+
|
6
|
+
def initialize(name, attributes = {})
|
7
|
+
if attributes[:base_directory]
|
8
|
+
@path = name
|
9
|
+
@name = self.class.relative_filename(name, attributes[:base_directory])
|
10
|
+
elsif attributes[:path]
|
11
|
+
@name = name
|
12
|
+
@path = attributes[:path]
|
13
|
+
else
|
14
|
+
@name = @path = name
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.relative_filename(path, base_directory)
|
19
|
+
::File.expand_path(path).gsub(%r{^#{::File.expand_path(base_directory)}/},"")
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_s
|
23
|
+
name
|
24
|
+
end
|
25
|
+
|
26
|
+
def basename
|
27
|
+
::File.basename(name, ".#{extension}")
|
28
|
+
end
|
29
|
+
|
30
|
+
def extension
|
31
|
+
::File.extname(name).gsub(/^\./,'')
|
32
|
+
end
|
33
|
+
|
34
|
+
def ==(other)
|
35
|
+
other and path == other.path
|
36
|
+
end
|
37
|
+
|
38
|
+
def match(expression)
|
39
|
+
name.match(expression).tap do |result|
|
40
|
+
verb = result ? "match" : "doesn't match"
|
41
|
+
Rivendell::Import.logger.debug "File #{verb} '#{expression}'"
|
42
|
+
!!result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def in(directory, &block)
|
47
|
+
if match %r{^#{directory}/}
|
48
|
+
yield
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def exists?
|
53
|
+
::File.exists? path
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Rivendell::Import
|
2
|
+
class Notification < ActiveRecord::Base
|
3
|
+
|
4
|
+
belongs_to :task
|
5
|
+
belongs_to :notifier, :class_name => "Rivendell::Import::Notifier::Base"
|
6
|
+
|
7
|
+
def sent?
|
8
|
+
sent_at.present?
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.pending
|
12
|
+
where :sent_at => nil
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'mail'
|
2
|
+
require 'erb'
|
3
|
+
|
4
|
+
module Rivendell::Import::Notifier
|
5
|
+
class Base < ActiveRecord::Base
|
6
|
+
|
7
|
+
# self.abstract_class = true
|
8
|
+
self.table_name = "notifiers"
|
9
|
+
|
10
|
+
def parameters
|
11
|
+
{}
|
12
|
+
end
|
13
|
+
|
14
|
+
has_many :notifications, :foreign_key => "notifier_id" do
|
15
|
+
def to_sent
|
16
|
+
pending.joins(:task).where("tasks.status" => %w{completed failed})
|
17
|
+
end
|
18
|
+
def waiting_for_more_than(delay)
|
19
|
+
joins(:task).where("tasks.updated_at < ?", Time.now - delay)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
has_many :tasks, :through => :notifications
|
24
|
+
|
25
|
+
def delay
|
26
|
+
tasks.pending.empty? ? 0 : 60
|
27
|
+
end
|
28
|
+
|
29
|
+
def notify
|
30
|
+
to_sent_notifications = notifications.to_sent.includes(:task)
|
31
|
+
|
32
|
+
if delay > 0
|
33
|
+
to_sent_notifications = to_sent_notifications.waiting_for_more_than(delay)
|
34
|
+
end
|
35
|
+
|
36
|
+
if to_sent_notifications.present?
|
37
|
+
logger.debug "Notify #{to_sent_notifications.size} tasks with #{self.inspect}"
|
38
|
+
notify! to_sent_notifications.map(&:task)
|
39
|
+
to_sent_notifications.update_all(:sent_at => Time.now)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def logger
|
44
|
+
Rivendell::Import.logger
|
45
|
+
end
|
46
|
+
|
47
|
+
def parameters=(parameters)
|
48
|
+
parameters.each { |k,v| send "#{k}=", v }
|
49
|
+
end
|
50
|
+
|
51
|
+
def raw_parameters
|
52
|
+
read_attribute :parameters
|
53
|
+
end
|
54
|
+
|
55
|
+
def read_parameters
|
56
|
+
self.parameters = ActiveSupport::JSON.decode(raw_parameters) if raw_parameters.present?
|
57
|
+
end
|
58
|
+
|
59
|
+
after_initialize :read_parameters
|
60
|
+
|
61
|
+
def write_parameters
|
62
|
+
if parameters.present?
|
63
|
+
write_attribute :parameters, parameters.to_json
|
64
|
+
write_attribute :key, parameters.hash
|
65
|
+
else
|
66
|
+
write_attribute :parameters, nil
|
67
|
+
write_attribute :key, nil
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
before_save :write_parameters
|
72
|
+
|
73
|
+
def self.notify(target, options = {})
|
74
|
+
new_notifier = case options.delete(:by)
|
75
|
+
when :email
|
76
|
+
Mail.new options.merge(:to => target)
|
77
|
+
end
|
78
|
+
|
79
|
+
key = new_notifier.parameters.hash
|
80
|
+
if existing_notifier = where(:type => new_notifier.type, :key => key).first
|
81
|
+
existing_notifier
|
82
|
+
else
|
83
|
+
new_notifier.save!
|
84
|
+
new_notifier
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|