rivendell-import 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/.gitignore +21 -0
  2. data/Gemfile +11 -0
  3. data/Guardfile +14 -0
  4. data/LICENSE +22 -0
  5. data/README.md +29 -0
  6. data/Rakefile +4 -0
  7. data/bin/rivendell-import +6 -0
  8. data/db/migrate/20120920712200_create_tasks.rb +21 -0
  9. data/db/migrate/20120927194300_create_notifiers.rb +14 -0
  10. data/db/migrate/20120927203600_create_notifications.rb +14 -0
  11. data/examples/.gitignore +2 -0
  12. data/examples/config.rb +31 -0
  13. data/features/manage_cart_attributes.feature +20 -0
  14. data/features/manage_file_matching.feature +44 -0
  15. data/features/step_definitions/import_steps.rb +38 -0
  16. data/features/support/env.rb +19 -0
  17. data/features/support/mock_xport.rb +34 -0
  18. data/lib/rivendell/import.rb +48 -0
  19. data/lib/rivendell/import/base.rb +57 -0
  20. data/lib/rivendell/import/cart.rb +67 -0
  21. data/lib/rivendell/import/carts_cache.rb +58 -0
  22. data/lib/rivendell/import/cli.rb +90 -0
  23. data/lib/rivendell/import/config.rb +13 -0
  24. data/lib/rivendell/import/context.rb +28 -0
  25. data/lib/rivendell/import/cut.rb +33 -0
  26. data/lib/rivendell/import/file.rb +57 -0
  27. data/lib/rivendell/import/notification.rb +16 -0
  28. data/lib/rivendell/import/notifier/base.rb +89 -0
  29. data/lib/rivendell/import/notifier/mail-body.erb +8 -0
  30. data/lib/rivendell/import/notifier/mail-subject.erb +11 -0
  31. data/lib/rivendell/import/notifier/mail.rb +104 -0
  32. data/lib/rivendell/import/notifiers.rb +24 -0
  33. data/lib/rivendell/import/task.rb +80 -0
  34. data/lib/rivendell/import/tasking/cart.rb +25 -0
  35. data/lib/rivendell/import/tasking/destination.rb +28 -0
  36. data/lib/rivendell/import/tasking/file.rb +20 -0
  37. data/lib/rivendell/import/tasking/status.rb +29 -0
  38. data/lib/rivendell/import/tasking/tags.rb +27 -0
  39. data/lib/rivendell/import/tasks.rb +23 -0
  40. data/lib/rivendell/import/version.rb +5 -0
  41. data/lib/rivendell/import/worker.rb +34 -0
  42. data/rivendell-import.gemspec +37 -0
  43. data/spec/fixtures/mail-body.erb +5 -0
  44. data/spec/rivendell/import/base_spec.rb +125 -0
  45. data/spec/rivendell/import/cart_spec.rb +132 -0
  46. data/spec/rivendell/import/carts_cache_spec.rb +110 -0
  47. data/spec/rivendell/import/cli_spec.rb +149 -0
  48. data/spec/rivendell/import/config_spec.rb +16 -0
  49. data/spec/rivendell/import/context_spec.rb +19 -0
  50. data/spec/rivendell/import/file_spec.rb +63 -0
  51. data/spec/rivendell/import/notifier/base_spec.rb +63 -0
  52. data/spec/rivendell/import/notifier/mail_spec.rb +110 -0
  53. data/spec/rivendell/import/task_spec.rb +217 -0
  54. data/spec/rivendell/import/tasks_spec.rb +30 -0
  55. data/spec/rivendell/import/worker_spec.rb +25 -0
  56. data/spec/rivendell/import_spec.rb +32 -0
  57. data/spec/spec_helper.rb +15 -0
  58. data/spec/support/database_cleaner.rb +17 -0
  59. data/spec/support/fixtures.rb +3 -0
  60. data/spec/support/mail.rb +3 -0
  61. data/spec/support/test_notifier.rb +15 -0
  62. data/tasks/ci.rake +2 -0
  63. data/tasks/cucumber.rake +4 -0
  64. data/tasks/database.rake +11 -0
  65. data/tasks/rdoc.rake +16 -0
  66. data/tasks/rspec.rake +2 -0
  67. metadata +399 -0
@@ -0,0 +1,67 @@
1
+ module Rivendell::Import
2
+ class Cart
3
+
4
+ include ActiveModel::Serialization
5
+ include ActiveModel::Serializers::JSON
6
+
7
+ def attributes
8
+ %w{number group}.inject({}) do |map, attribute|
9
+ value = send attribute
10
+ map[attribute] = value if value
11
+ map
12
+ end
13
+ end
14
+
15
+ def attributes=(attributes)
16
+ attributes.each { |k,v| send "#{k}=", v }
17
+ end
18
+
19
+ delegate :blank?, :to => :attributes
20
+
21
+ attr_accessor :number, :group
22
+ attr_reader :task
23
+
24
+ def initialize(task = nil)
25
+ @task = task
26
+ end
27
+
28
+ def xport
29
+ task.xport
30
+ end
31
+
32
+ def create
33
+ unless number
34
+ raise "Can't create Cart, Group isn't defined" unless group.present?
35
+ self.number = xport.add_cart(:group => group).number
36
+ end
37
+ end
38
+
39
+ def update
40
+ # xport.edit_cart # to define title, etc ...
41
+ end
42
+
43
+ def cut
44
+ @cut ||= Cut.new(self)
45
+ end
46
+
47
+ def import(file)
48
+ raise "File #{file.path} not found" unless file.exists?
49
+
50
+ cut.create
51
+ xport.import number, cut.number, file.path
52
+ cut.update
53
+ end
54
+
55
+ def find_by_title(string, options = {})
56
+ if remote_cart = carts_cache.find_by_title(string, options)
57
+ self.number = remote_cart.number
58
+ end
59
+ end
60
+
61
+ def carts_cache
62
+ @carts_cache ||= Rivendell::Import::CartsCache.new(xport)
63
+ end
64
+
65
+ end
66
+ end
67
+
@@ -0,0 +1,58 @@
1
+ module Rivendell::Import
2
+ class CartsCache
3
+
4
+ attr_accessor :xport
5
+
6
+ def initialize(xport)
7
+ @xport = xport
8
+ end
9
+
10
+ def find_all_by_title(string, options = {})
11
+ normalizer = options.delete(:normalizer) || Proc.new { |id| id }
12
+
13
+ string = normalizer.call(string)
14
+ carts(options).select do |cart|
15
+ normalizer.call(cart.title) == string
16
+ end
17
+ end
18
+
19
+ def default_normalizer
20
+ Proc.new do |string|
21
+ string.downcase.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
22
+ end
23
+ end
24
+
25
+ def find_by_title(string, options = {})
26
+ matching_carts = find_all_by_title(string, options)
27
+
28
+ if matching_carts.blank?
29
+ matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
30
+ end
31
+
32
+ matching_carts.first if matching_carts.one?
33
+ end
34
+
35
+ def cache
36
+ clear if purged_at < time_to_live.ago
37
+ @cache ||= {}
38
+ end
39
+
40
+ def clear
41
+ @cache = nil
42
+ end
43
+
44
+ @@time_to_live = 600
45
+ cattr_accessor :time_to_live
46
+
47
+ attr_accessor :purged_at
48
+
49
+ def purged_at
50
+ @purged_at ||= Time.now
51
+ end
52
+
53
+ def carts(options = {})
54
+ cache[options.to_s] ||= xport.list_carts(options)
55
+ end
56
+
57
+ end
58
+ end
@@ -0,0 +1,90 @@
1
+ require 'trollop'
2
+ require 'logger'
3
+
4
+ module Rivendell::Import
5
+ class CLI
6
+
7
+ attr_reader :arguments
8
+ attr_accessor :options_count
9
+
10
+ def initialize(arguments = [])
11
+ @arguments = arguments
12
+ end
13
+
14
+ def config_file
15
+ @config_file ||= options[:config]
16
+ end
17
+
18
+ def listen_mode?
19
+ options[:listen]
20
+ end
21
+
22
+ def dry_run?
23
+ options[:dry_run]
24
+ end
25
+
26
+ def debug?
27
+ options[:debug]
28
+ end
29
+
30
+ def database
31
+ options[:database]
32
+ end
33
+
34
+ def parser
35
+ @parser ||= Trollop::Parser.new do
36
+ opt :config, "Configuration file", :type => String
37
+ opt :listen, "Wait for files in given directory"
38
+ opt :dry_run, "Just create tasks without executing them"
39
+ opt :debug, "Enable debug messages (in stderr)"
40
+ opt :database, "The database file used to store tasks", :type => String
41
+ end
42
+ end
43
+
44
+ def options
45
+ @options ||= Trollop::with_standard_exception_handling(parser) do
46
+ raise Trollop::HelpNeeded if ARGV.empty? # show help screen
47
+ parser.parse arguments
48
+ end
49
+ end
50
+ alias_method :parse, :options
51
+
52
+ def parsed_parser
53
+ parse
54
+ parser
55
+ end
56
+
57
+ def import
58
+ @import ||= Rivendell::Import::Base.new
59
+ end
60
+
61
+ def paths
62
+ parsed_parser.leftovers
63
+ end
64
+
65
+ def run
66
+ Rivendell::Import.logger = Logger.new($stderr) if debug?
67
+
68
+ if database
69
+ Rivendell::Import.establish_connection database
70
+ else
71
+ Rivendell::Import.establish_connection
72
+ end
73
+
74
+ if config_file
75
+ load config_file
76
+ import.to_prepare = Rivendell::Import.config.to_prepare
77
+ end
78
+
79
+ if listen_mode?
80
+ listen_options = {}
81
+ listen_options[:dry_run] = true if dry_run?
82
+
83
+ import.listen paths.first, listen_options
84
+ else
85
+ import.process paths
86
+ import.tasks.run unless dry_run?
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,13 @@
1
+ module Rivendell::Import
2
+ class Config
3
+
4
+ def to_prepare(&block)
5
+ if block_given?
6
+ @prepare = block
7
+ else
8
+ @prepare
9
+ end
10
+ end
11
+
12
+ end
13
+ end
@@ -0,0 +1,28 @@
1
+ module Rivendell::Import
2
+ class Context
3
+
4
+ attr_reader :task
5
+
6
+ def initialize(task)
7
+ @task = task
8
+ end
9
+
10
+ delegate :file, :cart, :logger, :to => :task
11
+
12
+ def with(expression)
13
+ yield if file.match expression
14
+ end
15
+
16
+ def notify(target, options = {})
17
+ Rivendell::Import::Notifier::Base.notify(target, options).tap do |notifier|
18
+ logger.debug "Will notify with #{notifier.inspect}"
19
+ task.notifiers << notifier
20
+ end
21
+ end
22
+
23
+ def run(&block)
24
+ instance_exec file, &block if block_given?
25
+ end
26
+
27
+ end
28
+ end
@@ -0,0 +1,33 @@
1
+ module Rivendell::Import
2
+ class Cut
3
+
4
+ include ActiveModel::Serialization
5
+ include ActiveModel::Serializers::JSON
6
+
7
+ attr_reader :cart
8
+
9
+ attr_accessor :number
10
+
11
+ def initialize(cart)
12
+ @cart = cart
13
+ end
14
+
15
+ def attributes
16
+ {}
17
+ end
18
+
19
+ def xport
20
+ cart.xport
21
+ end
22
+
23
+ def create
24
+ # xport.delete_cuts # if clean cuts is required
25
+ self.number = xport.add_cut(cart.number).number unless number
26
+ end
27
+
28
+ def update
29
+ # xport.edit_cut # to define attributes
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,57 @@
1
+ module Rivendell::Import
2
+ class File
3
+
4
+ attr_reader :name, :path
5
+
6
+ def initialize(name, attributes = {})
7
+ if attributes[:base_directory]
8
+ @path = name
9
+ @name = self.class.relative_filename(name, attributes[:base_directory])
10
+ elsif attributes[:path]
11
+ @name = name
12
+ @path = attributes[:path]
13
+ else
14
+ @name = @path = name
15
+ end
16
+ end
17
+
18
+ def self.relative_filename(path, base_directory)
19
+ ::File.expand_path(path).gsub(%r{^#{::File.expand_path(base_directory)}/},"")
20
+ end
21
+
22
+ def to_s
23
+ name
24
+ end
25
+
26
+ def basename
27
+ ::File.basename(name, ".#{extension}")
28
+ end
29
+
30
+ def extension
31
+ ::File.extname(name).gsub(/^\./,'')
32
+ end
33
+
34
+ def ==(other)
35
+ other and path == other.path
36
+ end
37
+
38
+ def match(expression)
39
+ name.match(expression).tap do |result|
40
+ verb = result ? "match" : "doesn't match"
41
+ Rivendell::Import.logger.debug "File #{verb} '#{expression}'"
42
+ !!result
43
+ end
44
+ end
45
+
46
+ def in(directory, &block)
47
+ if match %r{^#{directory}/}
48
+ yield
49
+ end
50
+ end
51
+
52
+ def exists?
53
+ ::File.exists? path
54
+ end
55
+
56
+ end
57
+ end
@@ -0,0 +1,16 @@
1
+ module Rivendell::Import
2
+ class Notification < ActiveRecord::Base
3
+
4
+ belongs_to :task
5
+ belongs_to :notifier, :class_name => "Rivendell::Import::Notifier::Base"
6
+
7
+ def sent?
8
+ sent_at.present?
9
+ end
10
+
11
+ def self.pending
12
+ where :sent_at => nil
13
+ end
14
+
15
+ end
16
+ end
@@ -0,0 +1,89 @@
1
+ require 'mail'
2
+ require 'erb'
3
+
4
+ module Rivendell::Import::Notifier
5
+ class Base < ActiveRecord::Base
6
+
7
+ # self.abstract_class = true
8
+ self.table_name = "notifiers"
9
+
10
+ def parameters
11
+ {}
12
+ end
13
+
14
+ has_many :notifications, :foreign_key => "notifier_id" do
15
+ def to_sent
16
+ pending.joins(:task).where("tasks.status" => %w{completed failed})
17
+ end
18
+ def waiting_for_more_than(delay)
19
+ joins(:task).where("tasks.updated_at < ?", Time.now - delay)
20
+ end
21
+ end
22
+
23
+ has_many :tasks, :through => :notifications
24
+
25
+ def delay
26
+ tasks.pending.empty? ? 0 : 60
27
+ end
28
+
29
+ def notify
30
+ to_sent_notifications = notifications.to_sent.includes(:task)
31
+
32
+ if delay > 0
33
+ to_sent_notifications = to_sent_notifications.waiting_for_more_than(delay)
34
+ end
35
+
36
+ if to_sent_notifications.present?
37
+ logger.debug "Notify #{to_sent_notifications.size} tasks with #{self.inspect}"
38
+ notify! to_sent_notifications.map(&:task)
39
+ to_sent_notifications.update_all(:sent_at => Time.now)
40
+ end
41
+ end
42
+
43
+ def logger
44
+ Rivendell::Import.logger
45
+ end
46
+
47
+ def parameters=(parameters)
48
+ parameters.each { |k,v| send "#{k}=", v }
49
+ end
50
+
51
+ def raw_parameters
52
+ read_attribute :parameters
53
+ end
54
+
55
+ def read_parameters
56
+ self.parameters = ActiveSupport::JSON.decode(raw_parameters) if raw_parameters.present?
57
+ end
58
+
59
+ after_initialize :read_parameters
60
+
61
+ def write_parameters
62
+ if parameters.present?
63
+ write_attribute :parameters, parameters.to_json
64
+ write_attribute :key, parameters.hash
65
+ else
66
+ write_attribute :parameters, nil
67
+ write_attribute :key, nil
68
+ end
69
+ end
70
+
71
+ before_save :write_parameters
72
+
73
+ def self.notify(target, options = {})
74
+ new_notifier = case options.delete(:by)
75
+ when :email
76
+ Mail.new options.merge(:to => target)
77
+ end
78
+
79
+ key = new_notifier.parameters.hash
80
+ if existing_notifier = where(:type => new_notifier.type, :key => key).first
81
+ existing_notifier
82
+ else
83
+ new_notifier.save!
84
+ new_notifier
85
+ end
86
+ end
87
+
88
+ end
89
+ end