migratrix 0.0.9 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. data/lib/migratrix.rb +62 -6
  2. data/lib/migratrix/exceptions.rb +4 -1
  3. data/lib/migratrix/{extractors → extractions}/active_record.rb +14 -10
  4. data/lib/migratrix/{extractors/extractor.rb → extractions/extraction.rb} +21 -20
  5. data/lib/migratrix/loads/load.rb +43 -0
  6. data/lib/migratrix/loads/yaml.rb +15 -0
  7. data/lib/migratrix/migration.rb +115 -27
  8. data/lib/migratrix/migratrix.rb +43 -84
  9. data/lib/migratrix/registry.rb +20 -0
  10. data/lib/migratrix/transforms/map.rb +57 -0
  11. data/lib/migratrix/transforms/transform.rb +268 -0
  12. data/lib/migratrix/valid_options.rb +22 -0
  13. data/lib/patches/object_ext.rb +0 -4
  14. data/spec/fixtures/migrations/marbles_migration.rb +6 -4
  15. data/spec/lib/migratrix/{loggable_spec.rb → _loggable_spec.rb} +0 -0
  16. data/spec/lib/migratrix/extractions/active_record_spec.rb +146 -0
  17. data/spec/lib/migratrix/extractions/extraction_spec.rb +71 -0
  18. data/spec/lib/migratrix/loads/load_spec.rb +59 -0
  19. data/spec/lib/migratrix/loads/yaml_spec.rb +39 -0
  20. data/spec/lib/migratrix/migration_spec.rb +195 -27
  21. data/spec/lib/migratrix/migratrix_spec.rb +57 -85
  22. data/spec/lib/migratrix/registry_spec.rb +28 -0
  23. data/spec/lib/migratrix/transforms/map_spec.rb +55 -0
  24. data/spec/lib/migratrix/transforms/transform_spec.rb +134 -0
  25. data/spec/lib/migratrix_spec.rb +98 -0
  26. data/spec/lib/patches/object_ext_spec.rb +0 -7
  27. data/spec/spec_helper.rb +18 -13
  28. metadata +21 -10
  29. data/spec/lib/migratrix/extractors/active_record_spec.rb +0 -43
  30. data/spec/lib/migratrix/extractors/extractor_spec.rb +0 -63
  31. data/spec/lib/migratrix_module_spec.rb +0 -63
data/lib/migratrix.rb CHANGED
@@ -6,19 +6,75 @@ module Migratrix
6
6
  APP=Pathname.new(__FILE__).dirname + "migratrix"
7
7
  EXT=Pathname.new(__FILE__).dirname + "patches"
8
8
 
9
- def self.default_migrations_path
10
- Rails.root + 'db/legacy'
11
- end
12
-
13
9
  require EXT + 'string_ext'
14
10
  require EXT + 'object_ext'
15
11
  require EXT + 'andand'
16
12
  require APP + 'loggable'
13
+ require APP + 'valid_options'
17
14
  require APP + 'exceptions'
15
+ require APP + 'registry'
18
16
  require APP + 'migration'
19
17
  require APP + 'migratrix'
20
18
 
21
- require APP + 'extractors/extractor'
22
- require APP + 'extractors/active_record'
19
+ require APP + 'extractions/extraction'
20
+ require APP + 'extractions/active_record'
21
+
22
+ require APP + 'transforms/transform'
23
+ require APP + 'transforms/map'
24
+
25
+ require APP + 'loads/load'
26
+ require APP + 'loads/yaml'
27
+ # require APP + 'loads/csv'
28
+ # require APP + 'loads/active_record'
29
+
30
+
31
+ include ::Migratrix::Loggable
32
+
33
+ def self.register_extraction(name, klass, init_options={})
34
+ ::Migratrix::Migratrix.register_extraction(name, klass, init_options)
35
+ end
36
+
37
+ def self.extractions
38
+ ::Migratrix::Migratrix.extractions
39
+ end
40
+
41
+ def self.register_transform(name, klass, init_options={})
42
+ ::Migratrix::Migratrix.register_transform(name, klass, init_options)
43
+ end
44
+
45
+ def self.transforms
46
+ ::Migratrix::Migratrix.transforms
47
+ end
48
+
49
+ def self.register_load(name, klass, init_options={})
50
+ ::Migratrix::Migratrix.register_load(name, klass, init_options)
51
+ end
52
+
53
+ def self.loads
54
+ ::Migratrix::Migratrix.loads
55
+ end
56
+
57
+ def self.logger
58
+ ::Migratrix::Migratrix.logger
59
+ end
60
+
61
+ def self.logger=(new_logger)
62
+ ::Migratrix::Migratrix.logger = new_logger
63
+ end
64
+
65
+ def self.log_to(stream)
66
+ ::Migratrix::Migratrix.log_to(stream)
67
+ end
68
+
69
+ # ----------------------------------------------------------------------
70
+ # Register in-gem Components
71
+ register_extraction :extraction, Extractions::Extraction
72
+ register_extraction :active_record, Extractions::ActiveRecord
73
+
74
+ register_transform :transform, Transforms::Transform
75
+ register_transform :map, Transforms::Map
76
+
77
+ register_load :load, Loads::Load
78
+ register_load :yaml, Loads::Yaml
23
79
  end
24
80
 
@@ -4,6 +4,9 @@ module Migratrix
4
4
  class MigrationAlreadyExists < Exception; end
5
5
  class MigrationFileNotFound < Exception; end
6
6
  class MigrationNotDefined < Exception; end
7
- class ExtractorSourceUndefined < Exception; end
7
+ class ExtractionNotDefined < Exception; end
8
+ class TransformNotDefined < Exception; end
9
+ class LoadNotDefined < Exception; end
10
+ class ExtractionSourceUndefined < Exception; end
8
11
  end
9
12
 
@@ -1,9 +1,8 @@
1
1
  module Migratrix
2
- module Extractors
3
- # Extractor that expects to be pointed at an ActiveRecord class.
4
- class ActiveRecord < Extractor
5
- # TODO: Raise TypeError in initialize unless Extractor source is
6
- # an ActiveRecord model
2
+ module Extractions
3
+ # Extraction that expects to be pointed at an ActiveRecord class.
4
+ class ActiveRecord < Extraction
5
+ set_valid_options :fetchall
7
6
 
8
7
  def source=(new_source)
9
8
  raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(new_source)
@@ -14,8 +13,8 @@ module Migratrix
14
13
  source.is_a?(Class) && source.ancestors.include?(::ActiveRecord::Base)
15
14
  end
16
15
 
17
- def obtain_source(source)
18
- raise ExtractorSourceUndefined unless source
16
+ def obtain_source(source, options={})
17
+ raise ExtractionSourceUndefined unless source
19
18
  raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(source)
20
19
  source
21
20
  end
@@ -41,12 +40,17 @@ module Migratrix
41
40
  if source.is_a?(::ActiveRecord::Relation)
42
41
  source.to_sql
43
42
  else
44
- source.handle_where(1).to_sql
43
+ handle_where(source, 1).to_sql
45
44
  end
46
45
  end
47
46
 
48
- def execute_extract(source)
49
- source.all
47
+ def execute_extract(src, options={})
48
+ return src.all if options['fetchall']
49
+ ret = if src.is_a?(::ActiveRecord::Relation)
50
+ src
51
+ else
52
+ handle_where(src, 1)
53
+ end
50
54
  end
51
55
  end
52
56
  end
@@ -1,35 +1,36 @@
1
- #require 'active_model/attribute_methods'
2
-
3
1
  module Migratrix
4
- module Extractors
2
+ module Extractions
5
3
  # base class for extraction
6
- class Extractor
4
+ class Extraction
7
5
  include ::Migratrix::Loggable
8
- # include ActiveModel::AttributeMethods
6
+ include ::Migratrix::ValidOptions
7
+
8
+ attr_accessor :name, :source, :options
9
9
 
10
- attr_accessor :source, :options
10
+ set_valid_options :limit, :offset, :order, :where
11
11
 
12
- def initialize(options={})
12
+ def initialize(name, options={})
13
13
  @options = options.deep_copy
14
14
  self.source = options[:source] if options[:source]
15
15
  end
16
16
 
17
17
  def extract(options={})
18
- options = @options.merge(options)
18
+ options = @options.merge(options).symbolize_keys
19
19
 
20
20
  # TODO: Raise error if self.abstract? DANGER/NOTE that this is
21
21
  # the "default strategy" for extraction, and may need to be
22
22
  # extracted to a strategy object.
23
23
 
24
- src = obtain_source(self.source)
25
- src = handle_where(src, options["where"]) if options["where"]
26
- src = handle_order(src, options["order"]) if options["order"]
27
- src = handle_limit(src, options["limit"]) if options["limit"]
28
- src = handle_offset(src, options["offset"]) if options["offset"]
29
- execute_extract(src)
24
+ src = obtain_source(self.source, options)
25
+ src = handle_where(src, options[:where]) if options[:where]
26
+ src = handle_order(src, options[:order]) if options[:order]
27
+ src = handle_limit(src, options[:limit]) if options[:limit]
28
+ src = handle_offset(src, options[:offset]) if options[:offset]
29
+ execute_extract(src, options)
30
30
  end
31
31
 
32
- # = extraction filter methods
32
+
33
+ # = extraction filter methods
33
34
  #
34
35
  # The handle_* methods receive a source and return a source and
35
36
  # must be chainable. For example, source might come in as an
@@ -45,11 +46,11 @@ module Migratrix
45
46
 
46
47
  # First step in extraction is to take the given source and turn
47
48
  # it into something that the filter chain can used. The
48
- # ActiveRecord extractor uses a legacy model class as its source
49
- # so it can simply return its source. A CSV or Yaml extractor
49
+ # ActiveRecord extraction uses a legacy model class as its source
50
+ # so it can simply return its source. A CSV or Yaml extraction
50
51
  # here might need to read the entire file contents and returns
51
52
  # the full, unfiltered data source.
52
- def obtain_source(source)
53
+ def obtain_source(source, options={})
53
54
  raise NotImplementedError
54
55
  end
55
56
 
@@ -74,13 +75,13 @@ module Migratrix
74
75
  end
75
76
 
76
77
  # Constructs the query, if applicable. May not exist or make
77
- # sense for non-SQL and/or non-ActiveRecord extractors.
78
+ # sense for non-SQL and/or non-ActiveRecord extractions.
78
79
  def to_query(source)
79
80
  raise NotImplementedError
80
81
  end
81
82
 
82
83
  # Execute the extraction and return the result set.
83
- def execute_extract(source)
84
+ def execute_extract(source, options={})
84
85
  raise NotImplementedError
85
86
  end
86
87
  end
@@ -0,0 +1,43 @@
1
+ module Migratrix
2
+ module Loads
3
+ class Load
4
+ include ::Migratrix::Loggable
5
+ include ::Migratrix::ValidOptions
6
+
7
+ attr_accessor :name, :options
8
+
9
+ set_valid_options :transform
10
+
11
+ def initialize(name, options={})
12
+ @name = name
13
+ @options = options.symbolize_keys
14
+ end
15
+
16
+ # Default strategy: call save() on every transformed_object.
17
+ def load(transformed_objects)
18
+ transformed_objects.each do |transformed_object|
19
+ transformed_object.save
20
+ end
21
+ end
22
+
23
+ # Name of the transform to use. If omitted, returns our name.
24
+ def transform
25
+ options[:transform] || name
26
+ end
27
+
28
+
29
+ # # Prepare for load. Here is where you might want to truncate
30
+ # # database tables, clear out target files, etc.
31
+ # def before_load
32
+ # raise NotImplementedError
33
+ # end
34
+
35
+ # # Clean up after load. If you opened a file pointer in
36
+ # # before_load, now's a good time to close it.
37
+ # # TODO: Use the active model hooks to do this
38
+ # def after_load
39
+ # raise NotImplementedError
40
+ # end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,15 @@
1
+ require 'yaml'
2
+
3
+ module Migratrix
4
+ module Loads
5
+ class Yaml < Load
6
+ set_valid_options :filename
7
+
8
+ def load(transformed_items)
9
+ File.open(options[:filename], 'w') do |file|
10
+ file.puts transformed_items.to_yaml
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -6,46 +6,132 @@ module Migratrix
6
6
  class Migration
7
7
  include ::Migratrix::Loggable
8
8
  include ActiveModel::AttributeMethods
9
+ include Migratrix::ValidOptions
9
10
 
10
- cattr_accessor :extractor
11
11
  attr_accessor :options
12
+ set_valid_options :console
12
13
 
13
14
  def initialize(options={})
14
- @options = options.deep_copy
15
+ @options = options.deep_copy.symbolize_keys
16
+ Migratrix.log_to($stdout) if @options[:console]
15
17
  end
16
18
 
17
- # Adds an extractor to the extractors chain.
18
- def self.set_extractor(name, options={})
19
- # klassify this name
20
- raise NotImplementedError.new("Migratrix currently only supports ActiveRecord extractor.") unless name == :active_record
21
- @@extractor = ::Migratrix::Extractors::ActiveRecord.new(options)
19
+ # TODO: Technically, we need to ask our extractions, transformers
20
+ # and loaders for THEIR valid options as well. limit, offset,
21
+ # order and where are all extraction-only options, and fetchall is
22
+ # an ActiveRecord-specific option
23
+ def self.valid_options
24
+ opts = super # wacky, I know, but the extended ValidOptions module is in the super chain. (I <3 Ruby)
25
+ if extractions
26
+ extractions.each do |name, extraction|
27
+ opts += extraction.valid_options
28
+ end
29
+ end
30
+ if transforms
31
+ transforms.each do |name, transform|
32
+ opts += transform.valid_options
33
+ end
34
+ end
35
+ # if loads
36
+ # loads.each do |name, load|
37
+ # opts += load.valid_options
38
+ # end
39
+ # end
40
+ opts.uniq.sort
22
41
  end
23
42
 
24
- def extractor
25
- @@extractor
43
+ # TODO: THIS IS HUGE DUPLICATION, REFACTOR REFACTOR REFACTOR
44
+
45
+ # extraction crap
46
+ def self.set_extraction(extraction_name, class_name, options={})
47
+ extractions[extraction_name] = Migratrix.extraction(class_name, extraction_name, options)
48
+ end
49
+
50
+ def self.extend_extraction(extraction_name, options={})
51
+ migration = ancestors.detect {|k| k.respond_to?(:extractions) && k.extractions[extraction_name]}
52
+ raise ExtractionNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % extraction_name) unless migration
53
+ extraction = migration.extractions[extraction_name]
54
+ extractions[extraction_name] = extraction.class.new(extraction_name, extraction.options.merge(options))
55
+ end
56
+
57
+ def self.extractions
58
+ @extractions ||= {}
59
+ end
60
+
61
+ def extractions
62
+ self.class.extractions
63
+ end
64
+
65
+ # transform crap
66
+ def self.set_transform(name, type, options={})
67
+ transforms[name] = Migratrix.transform(name, type, options)
68
+ end
69
+
70
+ def self.extend_transform(transform_name, options={})
71
+ migration = ancestors.detect {|k| k.respond_to?(:transforms) && k.transforms[transform_name]}
72
+ raise TransformNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % transform_name) unless migration
73
+ transform = migration.transforms[transform_name]
74
+ transforms[transform_name] = transform.class.new(transform_name, transform.options.merge(options))
26
75
  end
27
76
 
28
- # OKAY, NEW RULE: You get ONE Extractor per Migration. You're
29
- # allowed to have multiple transform/load chains to the
30
- # extraction, but extractors? ONE.
77
+ def self.transforms
78
+ @transforms ||= {}
79
+ end
80
+
81
+ def transforms
82
+ self.class.transforms
83
+ end
84
+
85
+ # load crap
86
+ def self.set_load(name, type, options={})
87
+ loads[name] = Migratrix.load(name, type, options)
88
+ end
89
+
90
+ def self.extend_load(load_name, options={})
91
+ migration = ancestors.detect {|k| k.respond_to?(:loads) && k.loads[load_name]}
92
+ raise LoadNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % load_name) unless migration
93
+ load = migration.loads[load_name]
94
+ loads[load_name] = load.class.new(load_name, load.options.merge(options))
95
+ end
96
+
97
+ def self.loads
98
+ @loads ||= {}
99
+ end
100
+
101
+ def loads
102
+ self.class.loads
103
+ end
31
104
 
32
- # default extraction method; simply assigns @extractor.extract to
33
- # @extracted_items. If you override this method, you should
34
- # populate @extracted_items if you want the default transform
35
- # and/or load to work correctly.
36
105
  def extract
37
- extractor.extract(options)
106
+ extracted_items = {}
107
+ extractions.each do |name, extraction|
108
+ extracted_items[name] = extraction.extract(options)
109
+ end
110
+ extracted_items
38
111
  end
39
112
 
40
- # Transforms source data into outputs
41
- def transform
42
- # run the chain of transforms
113
+ # Transforms source data into outputs. @transformed_items is a
114
+ # hash of name => transformed_items.
115
+ #
116
+ def transform(extracted_items)
117
+ transformed_items = { }
118
+ transforms.each do |name, transform|
119
+ transformed_items[transform.name] = transform.transform extracted_items[transform.extraction]
120
+ end
121
+ transformed_items
43
122
  end
44
123
 
45
124
  # Saves the migrated data by "loading" it into our database or
46
- # other data sink.
47
- def load
48
- # run the chain of loads
125
+ # other data sink. Loaders have their own names, and by default
126
+ # they depend on a transformed_items key of the same name, but you
127
+ # may override this behavior by setting :source => :name or
128
+ # possibly :source => [:name1, :name2, etc].
129
+ def load(transformed_items)
130
+ loaded_items = { }
131
+ loads.each do |name, load|
132
+ loaded_items[load.name] = load.load transformed_items[load.transform]
133
+ end
134
+ loaded_items
49
135
  end
50
136
 
51
137
  # Perform the migration
@@ -54,12 +140,14 @@ module Migratrix
54
140
  # strategy. YAGNI: Rails 3 lets us defer the querying until we get
55
141
  # to the transform step, and then it's batched for us under the
56
142
  # hood. ...assuming, of course, we change the ActiveRecord
57
- # extractor's execute_extract method to return source instead of
143
+ # extraction's execute_extract method to return source instead of
58
144
  # all, but now the
59
145
  def migrate
60
- @extracted_items = extract
61
- transform
62
- load
146
+ # This fn || @var API lets you write a method and either set the
147
+ # @var or return the value.
148
+ @extracted_items = extract || @extracted_items
149
+ @transformed_items = transform(@extracted_items) || @transformed_items
150
+ load @transformed_items
63
151
  end
64
152
  end
65
153
  end