rivendell-import 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. data/.gitignore +21 -0
  2. data/Gemfile +11 -0
  3. data/Guardfile +14 -0
  4. data/LICENSE +22 -0
  5. data/README.md +29 -0
  6. data/Rakefile +4 -0
  7. data/bin/rivendell-import +6 -0
  8. data/db/migrate/20120920712200_create_tasks.rb +21 -0
  9. data/db/migrate/20120927194300_create_notifiers.rb +14 -0
  10. data/db/migrate/20120927203600_create_notifications.rb +14 -0
  11. data/examples/.gitignore +2 -0
  12. data/examples/config.rb +31 -0
  13. data/features/manage_cart_attributes.feature +20 -0
  14. data/features/manage_file_matching.feature +44 -0
  15. data/features/step_definitions/import_steps.rb +38 -0
  16. data/features/support/env.rb +19 -0
  17. data/features/support/mock_xport.rb +34 -0
  18. data/lib/rivendell/import.rb +48 -0
  19. data/lib/rivendell/import/base.rb +57 -0
  20. data/lib/rivendell/import/cart.rb +67 -0
  21. data/lib/rivendell/import/carts_cache.rb +58 -0
  22. data/lib/rivendell/import/cli.rb +90 -0
  23. data/lib/rivendell/import/config.rb +13 -0
  24. data/lib/rivendell/import/context.rb +28 -0
  25. data/lib/rivendell/import/cut.rb +33 -0
  26. data/lib/rivendell/import/file.rb +57 -0
  27. data/lib/rivendell/import/notification.rb +16 -0
  28. data/lib/rivendell/import/notifier/base.rb +89 -0
  29. data/lib/rivendell/import/notifier/mail-body.erb +8 -0
  30. data/lib/rivendell/import/notifier/mail-subject.erb +11 -0
  31. data/lib/rivendell/import/notifier/mail.rb +104 -0
  32. data/lib/rivendell/import/notifiers.rb +24 -0
  33. data/lib/rivendell/import/task.rb +80 -0
  34. data/lib/rivendell/import/tasking/cart.rb +25 -0
  35. data/lib/rivendell/import/tasking/destination.rb +28 -0
  36. data/lib/rivendell/import/tasking/file.rb +20 -0
  37. data/lib/rivendell/import/tasking/status.rb +29 -0
  38. data/lib/rivendell/import/tasking/tags.rb +27 -0
  39. data/lib/rivendell/import/tasks.rb +23 -0
  40. data/lib/rivendell/import/version.rb +5 -0
  41. data/lib/rivendell/import/worker.rb +34 -0
  42. data/rivendell-import.gemspec +37 -0
  43. data/spec/fixtures/mail-body.erb +5 -0
  44. data/spec/rivendell/import/base_spec.rb +125 -0
  45. data/spec/rivendell/import/cart_spec.rb +132 -0
  46. data/spec/rivendell/import/carts_cache_spec.rb +110 -0
  47. data/spec/rivendell/import/cli_spec.rb +149 -0
  48. data/spec/rivendell/import/config_spec.rb +16 -0
  49. data/spec/rivendell/import/context_spec.rb +19 -0
  50. data/spec/rivendell/import/file_spec.rb +63 -0
  51. data/spec/rivendell/import/notifier/base_spec.rb +63 -0
  52. data/spec/rivendell/import/notifier/mail_spec.rb +110 -0
  53. data/spec/rivendell/import/task_spec.rb +217 -0
  54. data/spec/rivendell/import/tasks_spec.rb +30 -0
  55. data/spec/rivendell/import/worker_spec.rb +25 -0
  56. data/spec/rivendell/import_spec.rb +32 -0
  57. data/spec/spec_helper.rb +15 -0
  58. data/spec/support/database_cleaner.rb +17 -0
  59. data/spec/support/fixtures.rb +3 -0
  60. data/spec/support/mail.rb +3 -0
  61. data/spec/support/test_notifier.rb +15 -0
  62. data/tasks/ci.rake +2 -0
  63. data/tasks/cucumber.rake +4 -0
  64. data/tasks/database.rake +11 -0
  65. data/tasks/rdoc.rake +16 -0
  66. data/tasks/rspec.rake +2 -0
  67. metadata +399 -0
@@ -0,0 +1,67 @@
1
+ module Rivendell::Import
2
+ class Cart
3
+
4
+ include ActiveModel::Serialization
5
+ include ActiveModel::Serializers::JSON
6
+
7
+ def attributes
8
+ %w{number group}.inject({}) do |map, attribute|
9
+ value = send attribute
10
+ map[attribute] = value if value
11
+ map
12
+ end
13
+ end
14
+
15
+ def attributes=(attributes)
16
+ attributes.each { |k,v| send "#{k}=", v }
17
+ end
18
+
19
+ delegate :blank?, :to => :attributes
20
+
21
+ attr_accessor :number, :group
22
+ attr_reader :task
23
+
24
+ def initialize(task = nil)
25
+ @task = task
26
+ end
27
+
28
+ def xport
29
+ task.xport
30
+ end
31
+
32
+ def create
33
+ unless number
34
+ raise "Can't create Cart, Group isn't defined" unless group.present?
35
+ self.number = xport.add_cart(:group => group).number
36
+ end
37
+ end
38
+
39
+ def update
40
+ # xport.edit_cart # to define title, etc ...
41
+ end
42
+
43
+ def cut
44
+ @cut ||= Cut.new(self)
45
+ end
46
+
47
+ def import(file)
48
+ raise "File #{file.path} not found" unless file.exists?
49
+
50
+ cut.create
51
+ xport.import number, cut.number, file.path
52
+ cut.update
53
+ end
54
+
55
+ def find_by_title(string, options = {})
56
+ if remote_cart = carts_cache.find_by_title(string, options)
57
+ self.number = remote_cart.number
58
+ end
59
+ end
60
+
61
+ def carts_cache
62
+ @carts_cache ||= Rivendell::Import::CartsCache.new(xport)
63
+ end
64
+
65
+ end
66
+ end
67
+
@@ -0,0 +1,58 @@
1
+ module Rivendell::Import
2
+ class CartsCache
3
+
4
+ attr_accessor :xport
5
+
6
+ def initialize(xport)
7
+ @xport = xport
8
+ end
9
+
10
+ def find_all_by_title(string, options = {})
11
+ normalizer = options.delete(:normalizer) || Proc.new { |id| id }
12
+
13
+ string = normalizer.call(string)
14
+ carts(options).select do |cart|
15
+ normalizer.call(cart.title) == string
16
+ end
17
+ end
18
+
19
+ def default_normalizer
20
+ Proc.new do |string|
21
+ string.downcase.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
22
+ end
23
+ end
24
+
25
+ def find_by_title(string, options = {})
26
+ matching_carts = find_all_by_title(string, options)
27
+
28
+ if matching_carts.blank?
29
+ matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
30
+ end
31
+
32
+ matching_carts.first if matching_carts.one?
33
+ end
34
+
35
+ def cache
36
+ clear if purged_at < time_to_live.ago
37
+ @cache ||= {}
38
+ end
39
+
40
+ def clear
41
+ @cache = nil
42
+ end
43
+
44
+ @@time_to_live = 600
45
+ cattr_accessor :time_to_live
46
+
47
+ attr_accessor :purged_at
48
+
49
+ def purged_at
50
+ @purged_at ||= Time.now
51
+ end
52
+
53
+ def carts(options = {})
54
+ cache[options.to_s] ||= xport.list_carts(options)
55
+ end
56
+
57
+ end
58
+ end
@@ -0,0 +1,90 @@
1
+ require 'trollop'
2
+ require 'logger'
3
+
4
+ module Rivendell::Import
5
+ class CLI
6
+
7
+ attr_reader :arguments
8
+ attr_accessor :options_count
9
+
10
+ def initialize(arguments = [])
11
+ @arguments = arguments
12
+ end
13
+
14
+ def config_file
15
+ @config_file ||= options[:config]
16
+ end
17
+
18
+ def listen_mode?
19
+ options[:listen]
20
+ end
21
+
22
+ def dry_run?
23
+ options[:dry_run]
24
+ end
25
+
26
+ def debug?
27
+ options[:debug]
28
+ end
29
+
30
+ def database
31
+ options[:database]
32
+ end
33
+
34
+ def parser
35
+ @parser ||= Trollop::Parser.new do
36
+ opt :config, "Configuration file", :type => String
37
+ opt :listen, "Wait for files in given directory"
38
+ opt :dry_run, "Just create tasks without executing them"
39
+ opt :debug, "Enable debug messages (in stderr)"
40
+ opt :database, "The database file used to store tasks", :type => String
41
+ end
42
+ end
43
+
44
+ def options
45
+ @options ||= Trollop::with_standard_exception_handling(parser) do
46
+ raise Trollop::HelpNeeded if ARGV.empty? # show help screen
47
+ parser.parse arguments
48
+ end
49
+ end
50
+ alias_method :parse, :options
51
+
52
+ def parsed_parser
53
+ parse
54
+ parser
55
+ end
56
+
57
+ def import
58
+ @import ||= Rivendell::Import::Base.new
59
+ end
60
+
61
+ def paths
62
+ parsed_parser.leftovers
63
+ end
64
+
65
+ def run
66
+ Rivendell::Import.logger = Logger.new($stderr) if debug?
67
+
68
+ if database
69
+ Rivendell::Import.establish_connection database
70
+ else
71
+ Rivendell::Import.establish_connection
72
+ end
73
+
74
+ if config_file
75
+ load config_file
76
+ import.to_prepare = Rivendell::Import.config.to_prepare
77
+ end
78
+
79
+ if listen_mode?
80
+ listen_options = {}
81
+ listen_options[:dry_run] = true if dry_run?
82
+
83
+ import.listen paths.first, listen_options
84
+ else
85
+ import.process paths
86
+ import.tasks.run unless dry_run?
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,13 @@
1
+ module Rivendell::Import
2
+ class Config
3
+
4
+ def to_prepare(&block)
5
+ if block_given?
6
+ @prepare = block
7
+ else
8
+ @prepare
9
+ end
10
+ end
11
+
12
+ end
13
+ end
@@ -0,0 +1,28 @@
1
+ module Rivendell::Import
2
+ class Context
3
+
4
+ attr_reader :task
5
+
6
+ def initialize(task)
7
+ @task = task
8
+ end
9
+
10
+ delegate :file, :cart, :logger, :to => :task
11
+
12
+ def with(expression)
13
+ yield if file.match expression
14
+ end
15
+
16
+ def notify(target, options = {})
17
+ Rivendell::Import::Notifier::Base.notify(target, options).tap do |notifier|
18
+ logger.debug "Will notify with #{notifier.inspect}"
19
+ task.notifiers << notifier
20
+ end
21
+ end
22
+
23
+ def run(&block)
24
+ instance_exec file, &block if block_given?
25
+ end
26
+
27
+ end
28
+ end
@@ -0,0 +1,33 @@
1
+ module Rivendell::Import
2
+ class Cut
3
+
4
+ include ActiveModel::Serialization
5
+ include ActiveModel::Serializers::JSON
6
+
7
+ attr_reader :cart
8
+
9
+ attr_accessor :number
10
+
11
+ def initialize(cart)
12
+ @cart = cart
13
+ end
14
+
15
+ def attributes
16
+ {}
17
+ end
18
+
19
+ def xport
20
+ cart.xport
21
+ end
22
+
23
+ def create
24
+ # xport.delete_cuts # if clean cuts is required
25
+ self.number = xport.add_cut(cart.number).number unless number
26
+ end
27
+
28
+ def update
29
+ # xport.edit_cut # to define attributes
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,57 @@
1
+ module Rivendell::Import
2
+ class File
3
+
4
+ attr_reader :name, :path
5
+
6
+ def initialize(name, attributes = {})
7
+ if attributes[:base_directory]
8
+ @path = name
9
+ @name = self.class.relative_filename(name, attributes[:base_directory])
10
+ elsif attributes[:path]
11
+ @name = name
12
+ @path = attributes[:path]
13
+ else
14
+ @name = @path = name
15
+ end
16
+ end
17
+
18
+ def self.relative_filename(path, base_directory)
19
+ ::File.expand_path(path).gsub(%r{^#{::File.expand_path(base_directory)}/},"")
20
+ end
21
+
22
+ def to_s
23
+ name
24
+ end
25
+
26
+ def basename
27
+ ::File.basename(name, ".#{extension}")
28
+ end
29
+
30
+ def extension
31
+ ::File.extname(name).gsub(/^\./,'')
32
+ end
33
+
34
+ def ==(other)
35
+ other and path == other.path
36
+ end
37
+
38
+ def match(expression)
39
+ name.match(expression).tap do |result|
40
+ verb = result ? "match" : "doesn't match"
41
+ Rivendell::Import.logger.debug "File #{verb} '#{expression}'"
42
+ !!result
43
+ end
44
+ end
45
+
46
+ def in(directory, &block)
47
+ if match %r{^#{directory}/}
48
+ yield
49
+ end
50
+ end
51
+
52
+ def exists?
53
+ ::File.exists? path
54
+ end
55
+
56
+ end
57
+ end
@@ -0,0 +1,16 @@
1
+ module Rivendell::Import
2
+ class Notification < ActiveRecord::Base
3
+
4
+ belongs_to :task
5
+ belongs_to :notifier, :class_name => "Rivendell::Import::Notifier::Base"
6
+
7
+ def sent?
8
+ sent_at.present?
9
+ end
10
+
11
+ def self.pending
12
+ where :sent_at => nil
13
+ end
14
+
15
+ end
16
+ end
@@ -0,0 +1,89 @@
1
+ require 'mail'
2
+ require 'erb'
3
+
4
+ module Rivendell::Import::Notifier
5
+ class Base < ActiveRecord::Base
6
+
7
+ # self.abstract_class = true
8
+ self.table_name = "notifiers"
9
+
10
+ def parameters
11
+ {}
12
+ end
13
+
14
+ has_many :notifications, :foreign_key => "notifier_id" do
15
+ def to_sent
16
+ pending.joins(:task).where("tasks.status" => %w{completed failed})
17
+ end
18
+ def waiting_for_more_than(delay)
19
+ joins(:task).where("tasks.updated_at < ?", Time.now - delay)
20
+ end
21
+ end
22
+
23
+ has_many :tasks, :through => :notifications
24
+
25
+ def delay
26
+ tasks.pending.empty? ? 0 : 60
27
+ end
28
+
29
+ def notify
30
+ to_sent_notifications = notifications.to_sent.includes(:task)
31
+
32
+ if delay > 0
33
+ to_sent_notifications = to_sent_notifications.waiting_for_more_than(delay)
34
+ end
35
+
36
+ if to_sent_notifications.present?
37
+ logger.debug "Notify #{to_sent_notifications.size} tasks with #{self.inspect}"
38
+ notify! to_sent_notifications.map(&:task)
39
+ to_sent_notifications.update_all(:sent_at => Time.now)
40
+ end
41
+ end
42
+
43
+ def logger
44
+ Rivendell::Import.logger
45
+ end
46
+
47
+ def parameters=(parameters)
48
+ parameters.each { |k,v| send "#{k}=", v }
49
+ end
50
+
51
+ def raw_parameters
52
+ read_attribute :parameters
53
+ end
54
+
55
+ def read_parameters
56
+ self.parameters = ActiveSupport::JSON.decode(raw_parameters) if raw_parameters.present?
57
+ end
58
+
59
+ after_initialize :read_parameters
60
+
61
+ def write_parameters
62
+ if parameters.present?
63
+ write_attribute :parameters, parameters.to_json
64
+ write_attribute :key, parameters.hash
65
+ else
66
+ write_attribute :parameters, nil
67
+ write_attribute :key, nil
68
+ end
69
+ end
70
+
71
+ before_save :write_parameters
72
+
73
+ def self.notify(target, options = {})
74
+ new_notifier = case options.delete(:by)
75
+ when :email
76
+ Mail.new options.merge(:to => target)
77
+ end
78
+
79
+ key = new_notifier.parameters.hash
80
+ if existing_notifier = where(:type => new_notifier.type, :key => key).first
81
+ existing_notifier
82
+ else
83
+ new_notifier.save!
84
+ new_notifier
85
+ end
86
+ end
87
+
88
+ end
89
+ end