migratrix 0.0.9 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/migratrix.rb +62 -6
- data/lib/migratrix/exceptions.rb +4 -1
- data/lib/migratrix/{extractors → extractions}/active_record.rb +14 -10
- data/lib/migratrix/{extractors/extractor.rb → extractions/extraction.rb} +21 -20
- data/lib/migratrix/loads/load.rb +43 -0
- data/lib/migratrix/loads/yaml.rb +15 -0
- data/lib/migratrix/migration.rb +115 -27
- data/lib/migratrix/migratrix.rb +43 -84
- data/lib/migratrix/registry.rb +20 -0
- data/lib/migratrix/transforms/map.rb +57 -0
- data/lib/migratrix/transforms/transform.rb +268 -0
- data/lib/migratrix/valid_options.rb +22 -0
- data/lib/patches/object_ext.rb +0 -4
- data/spec/fixtures/migrations/marbles_migration.rb +6 -4
- data/spec/lib/migratrix/{loggable_spec.rb → _loggable_spec.rb} +0 -0
- data/spec/lib/migratrix/extractions/active_record_spec.rb +146 -0
- data/spec/lib/migratrix/extractions/extraction_spec.rb +71 -0
- data/spec/lib/migratrix/loads/load_spec.rb +59 -0
- data/spec/lib/migratrix/loads/yaml_spec.rb +39 -0
- data/spec/lib/migratrix/migration_spec.rb +195 -27
- data/spec/lib/migratrix/migratrix_spec.rb +57 -85
- data/spec/lib/migratrix/registry_spec.rb +28 -0
- data/spec/lib/migratrix/transforms/map_spec.rb +55 -0
- data/spec/lib/migratrix/transforms/transform_spec.rb +134 -0
- data/spec/lib/migratrix_spec.rb +98 -0
- data/spec/lib/patches/object_ext_spec.rb +0 -7
- data/spec/spec_helper.rb +18 -13
- metadata +21 -10
- data/spec/lib/migratrix/extractors/active_record_spec.rb +0 -43
- data/spec/lib/migratrix/extractors/extractor_spec.rb +0 -63
- data/spec/lib/migratrix_module_spec.rb +0 -63
data/lib/migratrix.rb
CHANGED
@@ -6,19 +6,75 @@ module Migratrix
|
|
6
6
|
APP=Pathname.new(__FILE__).dirname + "migratrix"
|
7
7
|
EXT=Pathname.new(__FILE__).dirname + "patches"
|
8
8
|
|
9
|
-
def self.default_migrations_path
|
10
|
-
Rails.root + 'db/legacy'
|
11
|
-
end
|
12
|
-
|
13
9
|
require EXT + 'string_ext'
|
14
10
|
require EXT + 'object_ext'
|
15
11
|
require EXT + 'andand'
|
16
12
|
require APP + 'loggable'
|
13
|
+
require APP + 'valid_options'
|
17
14
|
require APP + 'exceptions'
|
15
|
+
require APP + 'registry'
|
18
16
|
require APP + 'migration'
|
19
17
|
require APP + 'migratrix'
|
20
18
|
|
21
|
-
require APP + '
|
22
|
-
require APP + '
|
19
|
+
require APP + 'extractions/extraction'
|
20
|
+
require APP + 'extractions/active_record'
|
21
|
+
|
22
|
+
require APP + 'transforms/transform'
|
23
|
+
require APP + 'transforms/map'
|
24
|
+
|
25
|
+
require APP + 'loads/load'
|
26
|
+
require APP + 'loads/yaml'
|
27
|
+
# require APP + 'loads/csv'
|
28
|
+
# require APP + 'loads/active_record'
|
29
|
+
|
30
|
+
|
31
|
+
include ::Migratrix::Loggable
|
32
|
+
|
33
|
+
def self.register_extraction(name, klass, init_options={})
|
34
|
+
::Migratrix::Migratrix.register_extraction(name, klass, init_options)
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.extractions
|
38
|
+
::Migratrix::Migratrix.extractions
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.register_transform(name, klass, init_options={})
|
42
|
+
::Migratrix::Migratrix.register_transform(name, klass, init_options)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.transforms
|
46
|
+
::Migratrix::Migratrix.transforms
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.register_load(name, klass, init_options={})
|
50
|
+
::Migratrix::Migratrix.register_load(name, klass, init_options)
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.loads
|
54
|
+
::Migratrix::Migratrix.loads
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.logger
|
58
|
+
::Migratrix::Migratrix.logger
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.logger=(new_logger)
|
62
|
+
::Migratrix::Migratrix.logger = new_logger
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.log_to(stream)
|
66
|
+
::Migratrix::Migratrix.log_to(stream)
|
67
|
+
end
|
68
|
+
|
69
|
+
# ----------------------------------------------------------------------
|
70
|
+
# Register in-gem Components
|
71
|
+
register_extraction :extraction, Extractions::Extraction
|
72
|
+
register_extraction :active_record, Extractions::ActiveRecord
|
73
|
+
|
74
|
+
register_transform :transform, Transforms::Transform
|
75
|
+
register_transform :map, Transforms::Map
|
76
|
+
|
77
|
+
register_load :load, Loads::Load
|
78
|
+
register_load :yaml, Loads::Yaml
|
23
79
|
end
|
24
80
|
|
data/lib/migratrix/exceptions.rb
CHANGED
@@ -4,6 +4,9 @@ module Migratrix
|
|
4
4
|
class MigrationAlreadyExists < Exception; end
|
5
5
|
class MigrationFileNotFound < Exception; end
|
6
6
|
class MigrationNotDefined < Exception; end
|
7
|
-
class
|
7
|
+
class ExtractionNotDefined < Exception; end
|
8
|
+
class TransformNotDefined < Exception; end
|
9
|
+
class LoadNotDefined < Exception; end
|
10
|
+
class ExtractionSourceUndefined < Exception; end
|
8
11
|
end
|
9
12
|
|
@@ -1,9 +1,8 @@
|
|
1
1
|
module Migratrix
|
2
|
-
module
|
3
|
-
#
|
4
|
-
class ActiveRecord <
|
5
|
-
|
6
|
-
# an ActiveRecord model
|
2
|
+
module Extractions
|
3
|
+
# Extraction that expects to be pointed at an ActiveRecord class.
|
4
|
+
class ActiveRecord < Extraction
|
5
|
+
set_valid_options :fetchall
|
7
6
|
|
8
7
|
def source=(new_source)
|
9
8
|
raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(new_source)
|
@@ -14,8 +13,8 @@ module Migratrix
|
|
14
13
|
source.is_a?(Class) && source.ancestors.include?(::ActiveRecord::Base)
|
15
14
|
end
|
16
15
|
|
17
|
-
def obtain_source(source)
|
18
|
-
raise
|
16
|
+
def obtain_source(source, options={})
|
17
|
+
raise ExtractionSourceUndefined unless source
|
19
18
|
raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(source)
|
20
19
|
source
|
21
20
|
end
|
@@ -41,12 +40,17 @@ module Migratrix
|
|
41
40
|
if source.is_a?(::ActiveRecord::Relation)
|
42
41
|
source.to_sql
|
43
42
|
else
|
44
|
-
|
43
|
+
handle_where(source, 1).to_sql
|
45
44
|
end
|
46
45
|
end
|
47
46
|
|
48
|
-
def execute_extract(
|
49
|
-
|
47
|
+
def execute_extract(src, options={})
|
48
|
+
return src.all if options['fetchall']
|
49
|
+
ret = if src.is_a?(::ActiveRecord::Relation)
|
50
|
+
src
|
51
|
+
else
|
52
|
+
handle_where(src, 1)
|
53
|
+
end
|
50
54
|
end
|
51
55
|
end
|
52
56
|
end
|
@@ -1,35 +1,36 @@
|
|
1
|
-
#require 'active_model/attribute_methods'
|
2
|
-
|
3
1
|
module Migratrix
|
4
|
-
module
|
2
|
+
module Extractions
|
5
3
|
# base class for extraction
|
6
|
-
class
|
4
|
+
class Extraction
|
7
5
|
include ::Migratrix::Loggable
|
8
|
-
|
6
|
+
include ::Migratrix::ValidOptions
|
7
|
+
|
8
|
+
attr_accessor :name, :source, :options
|
9
9
|
|
10
|
-
|
10
|
+
set_valid_options :limit, :offset, :order, :where
|
11
11
|
|
12
|
-
def initialize(options={})
|
12
|
+
def initialize(name, options={})
|
13
13
|
@options = options.deep_copy
|
14
14
|
self.source = options[:source] if options[:source]
|
15
15
|
end
|
16
16
|
|
17
17
|
def extract(options={})
|
18
|
-
options = @options.merge(options)
|
18
|
+
options = @options.merge(options).symbolize_keys
|
19
19
|
|
20
20
|
# TODO: Raise error if self.abstract? DANGER/NOTE that this is
|
21
21
|
# the "default strategy" for extraction, and may need to be
|
22
22
|
# extracted to a strategy object.
|
23
23
|
|
24
|
-
src = obtain_source(self.source)
|
25
|
-
src = handle_where(src, options[
|
26
|
-
src = handle_order(src, options[
|
27
|
-
src = handle_limit(src, options[
|
28
|
-
src = handle_offset(src, options[
|
29
|
-
execute_extract(src)
|
24
|
+
src = obtain_source(self.source, options)
|
25
|
+
src = handle_where(src, options[:where]) if options[:where]
|
26
|
+
src = handle_order(src, options[:order]) if options[:order]
|
27
|
+
src = handle_limit(src, options[:limit]) if options[:limit]
|
28
|
+
src = handle_offset(src, options[:offset]) if options[:offset]
|
29
|
+
execute_extract(src, options)
|
30
30
|
end
|
31
31
|
|
32
|
-
|
32
|
+
|
33
|
+
# = extraction filter methods
|
33
34
|
#
|
34
35
|
# The handle_* methods receive a source and return a source and
|
35
36
|
# must be chainable. For example, source might come in as an
|
@@ -45,11 +46,11 @@ module Migratrix
|
|
45
46
|
|
46
47
|
# First step in extraction is to take the given source and turn
|
47
48
|
# it into something that the filter chain can used. The
|
48
|
-
# ActiveRecord
|
49
|
-
# so it can simply return its source. A CSV or Yaml
|
49
|
+
# ActiveRecord extraction uses a legacy model class as its source
|
50
|
+
# so it can simply return its source. A CSV or Yaml extraction
|
50
51
|
# here might need to read the entire file contents and returns
|
51
52
|
# the full, unfiltered data source.
|
52
|
-
def obtain_source(source)
|
53
|
+
def obtain_source(source, options={})
|
53
54
|
raise NotImplementedError
|
54
55
|
end
|
55
56
|
|
@@ -74,13 +75,13 @@ module Migratrix
|
|
74
75
|
end
|
75
76
|
|
76
77
|
# Constructs the query, if applicable. May not exist or make
|
77
|
-
# sense for non-SQL and/or non-ActiveRecord
|
78
|
+
# sense for non-SQL and/or non-ActiveRecord extractions.
|
78
79
|
def to_query(source)
|
79
80
|
raise NotImplementedError
|
80
81
|
end
|
81
82
|
|
82
83
|
# Execute the extraction and return the result set.
|
83
|
-
def execute_extract(source)
|
84
|
+
def execute_extract(source, options={})
|
84
85
|
raise NotImplementedError
|
85
86
|
end
|
86
87
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Migratrix
|
2
|
+
module Loads
|
3
|
+
class Load
|
4
|
+
include ::Migratrix::Loggable
|
5
|
+
include ::Migratrix::ValidOptions
|
6
|
+
|
7
|
+
attr_accessor :name, :options
|
8
|
+
|
9
|
+
set_valid_options :transform
|
10
|
+
|
11
|
+
def initialize(name, options={})
|
12
|
+
@name = name
|
13
|
+
@options = options.symbolize_keys
|
14
|
+
end
|
15
|
+
|
16
|
+
# Default strategy: call save() on every transformed_object.
|
17
|
+
def load(transformed_objects)
|
18
|
+
transformed_objects.each do |transformed_object|
|
19
|
+
transformed_object.save
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Name of the transform to use. If omitted, returns our name.
|
24
|
+
def transform
|
25
|
+
options[:transform] || name
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
# # Prepare for load. Here is where you might want to truncate
|
30
|
+
# # database tables, clear out target files, etc.
|
31
|
+
# def before_load
|
32
|
+
# raise NotImplementedError
|
33
|
+
# end
|
34
|
+
|
35
|
+
# # Clean up after load. If you opened a file pointer in
|
36
|
+
# # before_load, now's a good time to close it.
|
37
|
+
# # TODO: Use the active model hooks to do this
|
38
|
+
# def after_load
|
39
|
+
# raise NotImplementedError
|
40
|
+
# end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Migratrix
|
4
|
+
module Loads
|
5
|
+
class Yaml < Load
|
6
|
+
set_valid_options :filename
|
7
|
+
|
8
|
+
def load(transformed_items)
|
9
|
+
File.open(options[:filename], 'w') do |file|
|
10
|
+
file.puts transformed_items.to_yaml
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/migratrix/migration.rb
CHANGED
@@ -6,46 +6,132 @@ module Migratrix
|
|
6
6
|
class Migration
|
7
7
|
include ::Migratrix::Loggable
|
8
8
|
include ActiveModel::AttributeMethods
|
9
|
+
include Migratrix::ValidOptions
|
9
10
|
|
10
|
-
cattr_accessor :extractor
|
11
11
|
attr_accessor :options
|
12
|
+
set_valid_options :console
|
12
13
|
|
13
14
|
def initialize(options={})
|
14
|
-
@options = options.deep_copy
|
15
|
+
@options = options.deep_copy.symbolize_keys
|
16
|
+
Migratrix.log_to($stdout) if @options[:console]
|
15
17
|
end
|
16
18
|
|
17
|
-
#
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
# TODO: Technically, we need to ask our extractions, transformers
|
20
|
+
# and loaders for THEIR valid options as well. limit, offset,
|
21
|
+
# order and where are all extraction-only options, and fetchall is
|
22
|
+
# an ActiveRecord-specific option
|
23
|
+
def self.valid_options
|
24
|
+
opts = super # wacky, I know, but the extended ValidOptions module is in the super chain. (I <3 Ruby)
|
25
|
+
if extractions
|
26
|
+
extractions.each do |name, extraction|
|
27
|
+
opts += extraction.valid_options
|
28
|
+
end
|
29
|
+
end
|
30
|
+
if transforms
|
31
|
+
transforms.each do |name, transform|
|
32
|
+
opts += transform.valid_options
|
33
|
+
end
|
34
|
+
end
|
35
|
+
# if loads
|
36
|
+
# loads.each do |name, load|
|
37
|
+
# opts += load.valid_options
|
38
|
+
# end
|
39
|
+
# end
|
40
|
+
opts.uniq.sort
|
22
41
|
end
|
23
42
|
|
24
|
-
|
25
|
-
|
43
|
+
# TODO: THIS IS HUGE DUPLICATION, REFACTOR REFACTOR REFACTOR
|
44
|
+
|
45
|
+
# extraction crap
|
46
|
+
def self.set_extraction(extraction_name, class_name, options={})
|
47
|
+
extractions[extraction_name] = Migratrix.extraction(class_name, extraction_name, options)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.extend_extraction(extraction_name, options={})
|
51
|
+
migration = ancestors.detect {|k| k.respond_to?(:extractions) && k.extractions[extraction_name]}
|
52
|
+
raise ExtractionNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % extraction_name) unless migration
|
53
|
+
extraction = migration.extractions[extraction_name]
|
54
|
+
extractions[extraction_name] = extraction.class.new(extraction_name, extraction.options.merge(options))
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.extractions
|
58
|
+
@extractions ||= {}
|
59
|
+
end
|
60
|
+
|
61
|
+
def extractions
|
62
|
+
self.class.extractions
|
63
|
+
end
|
64
|
+
|
65
|
+
# transform crap
|
66
|
+
def self.set_transform(name, type, options={})
|
67
|
+
transforms[name] = Migratrix.transform(name, type, options)
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.extend_transform(transform_name, options={})
|
71
|
+
migration = ancestors.detect {|k| k.respond_to?(:transforms) && k.transforms[transform_name]}
|
72
|
+
raise TransformNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % transform_name) unless migration
|
73
|
+
transform = migration.transforms[transform_name]
|
74
|
+
transforms[transform_name] = transform.class.new(transform_name, transform.options.merge(options))
|
26
75
|
end
|
27
76
|
|
28
|
-
|
29
|
-
|
30
|
-
|
77
|
+
def self.transforms
|
78
|
+
@transforms ||= {}
|
79
|
+
end
|
80
|
+
|
81
|
+
def transforms
|
82
|
+
self.class.transforms
|
83
|
+
end
|
84
|
+
|
85
|
+
# load crap
|
86
|
+
def self.set_load(name, type, options={})
|
87
|
+
loads[name] = Migratrix.load(name, type, options)
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.extend_load(load_name, options={})
|
91
|
+
migration = ancestors.detect {|k| k.respond_to?(:loads) && k.loads[load_name]}
|
92
|
+
raise LoadNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % load_name) unless migration
|
93
|
+
load = migration.loads[load_name]
|
94
|
+
loads[load_name] = load.class.new(load_name, load.options.merge(options))
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.loads
|
98
|
+
@loads ||= {}
|
99
|
+
end
|
100
|
+
|
101
|
+
def loads
|
102
|
+
self.class.loads
|
103
|
+
end
|
31
104
|
|
32
|
-
# default extraction method; simply assigns @extractor.extract to
|
33
|
-
# @extracted_items. If you override this method, you should
|
34
|
-
# populate @extracted_items if you want the default transform
|
35
|
-
# and/or load to work correctly.
|
36
105
|
def extract
|
37
|
-
|
106
|
+
extracted_items = {}
|
107
|
+
extractions.each do |name, extraction|
|
108
|
+
extracted_items[name] = extraction.extract(options)
|
109
|
+
end
|
110
|
+
extracted_items
|
38
111
|
end
|
39
112
|
|
40
|
-
# Transforms source data into outputs
|
41
|
-
|
42
|
-
|
113
|
+
# Transforms source data into outputs. @transformed_items is a
|
114
|
+
# hash of name => transformed_items.
|
115
|
+
#
|
116
|
+
def transform(extracted_items)
|
117
|
+
transformed_items = { }
|
118
|
+
transforms.each do |name, transform|
|
119
|
+
transformed_items[transform.name] = transform.transform extracted_items[transform.extraction]
|
120
|
+
end
|
121
|
+
transformed_items
|
43
122
|
end
|
44
123
|
|
45
124
|
# Saves the migrated data by "loading" it into our database or
|
46
|
-
# other data sink.
|
47
|
-
|
48
|
-
|
125
|
+
# other data sink. Loaders have their own names, and by default
|
126
|
+
# they depend on a transformed_items key of the same name, but you
|
127
|
+
# may override this behavior by setting :source => :name or
|
128
|
+
# possibly :source => [:name1, :name2, etc].
|
129
|
+
def load(transformed_items)
|
130
|
+
loaded_items = { }
|
131
|
+
loads.each do |name, load|
|
132
|
+
loaded_items[load.name] = load.load transformed_items[load.transform]
|
133
|
+
end
|
134
|
+
loaded_items
|
49
135
|
end
|
50
136
|
|
51
137
|
# Perform the migration
|
@@ -54,12 +140,14 @@ module Migratrix
|
|
54
140
|
# strategy. YAGNI: Rails 3 lets us defer the querying until we get
|
55
141
|
# to the transform step, and then it's batched for us under the
|
56
142
|
# hood. ...assuming, of course, we change the ActiveRecord
|
57
|
-
#
|
143
|
+
# extraction's execute_extract method to return source instead of
|
58
144
|
# all, but now the
|
59
145
|
def migrate
|
60
|
-
@
|
61
|
-
|
62
|
-
|
146
|
+
# This fn || @var API lets you write a method and either set the
|
147
|
+
# @var or return the value.
|
148
|
+
@extracted_items = extract || @extracted_items
|
149
|
+
@transformed_items = transform(@extracted_items) || @transformed_items
|
150
|
+
load @transformed_items
|
63
151
|
end
|
64
152
|
end
|
65
153
|
end
|