caruby-core 1.5.3 → 1.5.4
Sign up to get free protection for your applications and to get access to all the features.
- data/History.md +4 -0
- data/lib/caruby/cli/command.rb +2 -1
- data/lib/caruby/database/reader.rb +2 -3
- data/lib/caruby/database/search_template_builder.rb +5 -5
- data/lib/caruby/database/sql_executor.rb +7 -3
- data/lib/caruby/database/writer.rb +8 -9
- data/lib/caruby/domain/attribute.rb +6 -3
- data/lib/caruby/domain/attributes.rb +6 -1
- data/lib/caruby/domain/importer.rb +3 -3
- data/lib/caruby/import/java.rb +18 -6
- data/lib/caruby/json/deserializer.rb +15 -0
- data/lib/caruby/json/serializer.rb +27 -0
- data/lib/caruby/migration/migratable.rb +3 -6
- data/lib/caruby/migration/migrator.rb +267 -91
- data/lib/caruby/resource.rb +5 -3
- data/lib/caruby/util/collection.rb +8 -15
- data/lib/caruby/util/log.rb +1 -1
- data/lib/caruby/util/properties.rb +1 -1
- data/lib/caruby/util/topological_sync_enumerator.rb +4 -1
- data/lib/caruby/util/transitive_closure.rb +19 -9
- data/lib/caruby/version.rb +1 -1
- data/test/lib/caruby/util/collection_test.rb +2 -6
- data/test/lib/caruby/util/transitive_closure_test.rb +21 -13
- data/test/lib/examples/clinical_trials/migration/migration_test.rb +58 -0
- data/test/lib/examples/clinical_trials/migration/test_case.rb +38 -0
- metadata +8 -6
- data/test/lib/examples/galena/clinical_trials/migration/participant_test.rb +0 -28
- data/test/lib/examples/galena/clinical_trials/migration/test_case.rb +0 -40
data/History.md
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
This history lists major release themes. See the GitHub Commits (https://github.com/caruby/core)
|
2
2
|
for change details.
|
3
3
|
|
4
|
+
1.5.4 / 2011-07-19
|
5
|
+
------------------
|
6
|
+
* Add migration value filter option.
|
7
|
+
|
4
8
|
1.5.3 / 2011-07-08
|
5
9
|
------------------
|
6
10
|
* Clean up documentation.
|
data/lib/caruby/cli/command.rb
CHANGED
@@ -32,11 +32,12 @@ module CaRuby
|
|
32
32
|
#
|
33
33
|
# Built-in options include the following:
|
34
34
|
# * --help : print the help message and exit
|
35
|
-
# * --
|
35
|
+
# * --verbose : print additional information to the console
|
36
36
|
# * --log FILE : log file
|
37
37
|
# * --debug : print debug messages to the log
|
38
38
|
# * --file FILE: configuration file containing other options
|
39
39
|
# * --quiet: suppress printing messages to stdout
|
40
|
+
#
|
40
41
|
# This class processes these built-in options, with the exception of +--version+,
|
41
42
|
# which is a subclass responsibility. Subclasses are responsible for
|
42
43
|
# processing any remaining options.
|
@@ -321,15 +321,14 @@ module CaRuby
|
|
321
321
|
return
|
322
322
|
end
|
323
323
|
@transients << obj
|
324
|
-
|
325
324
|
logger.debug { "Fetching #{obj.qp} from the database..." }
|
326
325
|
fetched = fetch_object(obj) || return
|
327
|
-
# fetch_object can return obj; if so, then done
|
326
|
+
# fetch_object can return obj; if so, then done, otherwise, merge fetched.
|
328
327
|
return obj if obj.equal?(fetched)
|
329
328
|
|
330
329
|
logger.debug { "Fetch #{obj.qp} matched database object #{fetched}." }
|
331
330
|
@transients.delete(obj)
|
332
|
-
#
|
331
|
+
# recursively copy the nondomain attributes, esp. the identifer, of the fetched domain object references
|
333
332
|
merge_fetched(fetched, obj)
|
334
333
|
# Inject the lazy loader for loadable domain reference attributes.
|
335
334
|
persistify(obj, fetched)
|
@@ -24,14 +24,14 @@ module CaRuby
|
|
24
24
|
logger.debug { "Building search template for #{obj.qp}..." }
|
25
25
|
hash ||= obj.value_hash(obj.class.searchable_attributes)
|
26
26
|
# the searchable attribute => value hash
|
27
|
-
|
27
|
+
rh, nrh = hash.split { |attr, value| Resource === value }
|
28
28
|
# make the search template from the non-reference attributes
|
29
|
-
tmpl = obj.class.new.merge_attributes(
|
29
|
+
tmpl = obj.class.new.merge_attributes(nrh)
|
30
30
|
# get references for the search template
|
31
|
-
unless
|
32
|
-
logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{
|
31
|
+
unless rh.empty? then
|
32
|
+
logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{rh.keys.to_series}..." }
|
33
33
|
end
|
34
|
-
|
34
|
+
rh.each { |attr, ref| add_search_template_reference(tmpl, ref, attr) }
|
35
35
|
tmpl
|
36
36
|
end
|
37
37
|
|
@@ -34,6 +34,9 @@ module CaRuby
|
|
34
34
|
# @option opts [String] :database_driver_class the optional DBI connect driver class name
|
35
35
|
# @raise [CaRuby::ConfigurationError] if an option is invalid
|
36
36
|
def initialize(opts)
|
37
|
+
if opts.empty? then
|
38
|
+
raise CaRuby::ConfigurationError.new("The caRuby database connection properties were not found.")
|
39
|
+
end
|
37
40
|
app_host = Options.get(:host, opts, 'localhost')
|
38
41
|
db_host = Options.get(:database_host, opts, app_host)
|
39
42
|
db_type = Options.get(:database_type, opts, 'mysql')
|
@@ -52,9 +55,10 @@ module CaRuby
|
|
52
55
|
:database_type => db_type,
|
53
56
|
:database_port => db_port,
|
54
57
|
:database_driver => db_driver,
|
55
|
-
:database_driver_class => @driver_class
|
56
|
-
|
57
|
-
|
58
|
+
:database_driver_class => @driver_class,
|
59
|
+
:address => @address
|
60
|
+
}
|
61
|
+
logger.debug { "Database connection parameters (excluding password): #{eff_opts.qp}" }
|
58
62
|
end
|
59
63
|
|
60
64
|
# Connects to the database, yields the DBI handle to the given block and disconnects.
|
@@ -139,6 +139,7 @@ module CaRuby
|
|
139
139
|
def ensure_exists(obj)
|
140
140
|
raise ArgumentError.new("Database ensure_exists is missing a domain object argument") if obj.nil_or_empty?
|
141
141
|
obj.enumerate { |ref| find(ref, :create) unless ref.identifier }
|
142
|
+
|
142
143
|
end
|
143
144
|
|
144
145
|
private
|
@@ -286,7 +287,6 @@ module CaRuby
|
|
286
287
|
logger.debug { "Ensuring that created #{obj.qp} references exist: #{refs.qp}..." } unless refs.empty?
|
287
288
|
refs.each { |ref| ensure_exists(ref) }
|
288
289
|
end
|
289
|
-
|
290
290
|
obj
|
291
291
|
end
|
292
292
|
|
@@ -350,7 +350,7 @@ module CaRuby
|
|
350
350
|
# * the attribute references a {#pending_create?} save context.
|
351
351
|
#
|
352
352
|
# @param obj (see #create)
|
353
|
-
# @param [Attribute] attr_md candidate attribute metadata
|
353
|
+
# @param [Attribute] attr_md the candidate attribute metadata
|
354
354
|
# @return [Boolean] whether the attribute should not be included in the create template
|
355
355
|
def exclude_pending_create_attribute?(obj, attr_md)
|
356
356
|
attr_md.independent? and
|
@@ -413,7 +413,7 @@ module CaRuby
|
|
413
413
|
# update a cascaded dependent by updating the owner
|
414
414
|
owner = cascaded_owner(obj)
|
415
415
|
if owner then return update_cascaded_dependent(owner, obj) end
|
416
|
-
|
416
|
+
|
417
417
|
# Not cascaded dependent; update using a template,
|
418
418
|
tmpl = build_update_template(obj)
|
419
419
|
# call the caCORE service with an obj update template
|
@@ -559,10 +559,9 @@ module CaRuby
|
|
559
559
|
logger.debug { "Updating saved #{target} to store unsaved attributes..." }
|
560
560
|
# call update_object(saved) rather than update(saved) to bypass the redundant update check
|
561
561
|
perform(:update, target) { update_object(target) }
|
562
|
-
else
|
563
|
-
# recursively save the dependents
|
564
|
-
save_changed_dependents(target)
|
565
562
|
end
|
563
|
+
# recursively save the dependents
|
564
|
+
save_changed_dependents(target)
|
566
565
|
end
|
567
566
|
|
568
567
|
# Synchronizes the given saved target result source with the database content.
|
@@ -685,9 +684,9 @@ module CaRuby
|
|
685
684
|
# @param [Resource] obj the owner domain object
|
686
685
|
def save_changed_dependents(obj)
|
687
686
|
obj.class.dependent_attributes.each do |attr|
|
688
|
-
deps = obj.send(attr)
|
689
|
-
logger.debug { "Saving the #{obj} #{attr} dependents #{deps.qp(:single_line)} which have changed..." } unless deps.
|
690
|
-
deps.
|
687
|
+
deps = obj.send(attr)
|
688
|
+
logger.debug { "Saving the #{obj} #{attr} dependents #{deps.to_enum.qp(:single_line)} which have changed..." } unless deps.nil_or_empty?
|
689
|
+
deps.enumerate { |dep| save_dependent_if_changed(obj, attr, dep) }
|
691
690
|
end
|
692
691
|
end
|
693
692
|
|
@@ -127,7 +127,7 @@ module CaRuby
|
|
127
127
|
# The inverse relation is symmetric, i.e. the inverse of the referenced Attribute
|
128
128
|
# is set to this Attribute's subject attribute.
|
129
129
|
#
|
130
|
-
# @param attribute the inverse attribute
|
130
|
+
# @param [Symbol, nil] attribute the inverse attribute
|
131
131
|
def inverse=(attribute)
|
132
132
|
return if inverse == attribute
|
133
133
|
# if no attribute, then the clear the existing inverse, if any
|
@@ -522,9 +522,12 @@ module CaRuby
|
|
522
522
|
def clear_inverse
|
523
523
|
return unless @inv_md
|
524
524
|
logger.debug { "Clearing #{@declarer.qp}.#{self} inverse #{type.qp}.#{inverse}..." }
|
525
|
-
|
525
|
+
# Capture the inverse before unsetting it.
|
526
|
+
inv_md = @inv_md
|
527
|
+
# Unset the inverse.
|
526
528
|
@inv_md = nil
|
527
|
-
|
529
|
+
# Clear the inverse of the inverse.
|
530
|
+
inv_md.inverse = nil
|
528
531
|
logger.debug { "Cleared #{@declarer.qp}.#{self} inverse." }
|
529
532
|
end
|
530
533
|
|
@@ -530,8 +530,13 @@ module CaRuby
|
|
530
530
|
def remove_attribute(attribute)
|
531
531
|
std_attr = standard_attribute(attribute)
|
532
532
|
# if the attribute is local, then delete it, otherwise filter out the superclass attribute
|
533
|
-
|
533
|
+
attr_md = @local_attr_md_hash.delete(std_attr)
|
534
|
+
if attr_md then
|
535
|
+
# clear the inverse, if any
|
536
|
+
attr_md.inverse = nil
|
537
|
+
# remove from the mandatory attributes, if necessary
|
534
538
|
@local_mndty_attrs.delete(std_attr)
|
539
|
+
# remove from the attribute => metadata hash
|
535
540
|
@local_std_attr_hash.delete_if { |aliaz, attr| attr == std_attr }
|
536
541
|
else
|
537
542
|
# Filter the superclass hashes.
|
@@ -39,8 +39,8 @@ module CaRuby
|
|
39
39
|
logger.debug { "Detecting whether #{symbol} is a #{@pkg} Java class..." }
|
40
40
|
# Append the symbol to the package to make the Java class name.
|
41
41
|
begin
|
42
|
-
klass =
|
43
|
-
resource_import
|
42
|
+
klass = java_import "#{@pkg}.#{symbol}"
|
43
|
+
resource_import(klass)
|
44
44
|
rescue NameError
|
45
45
|
logger.debug { "#{symbol} is not recognized as a #{@pkg} Java class - #{$!}\n#{caller.qp}." }
|
46
46
|
super
|
@@ -53,7 +53,7 @@ module CaRuby
|
|
53
53
|
# The Java class is assumed to be defined in this module's package.
|
54
54
|
# This module's mixin is added to the class.
|
55
55
|
#
|
56
|
-
# @param [
|
56
|
+
# @param [Class] klass the source directory
|
57
57
|
# @raise [NameError] if the symbol does not correspond to a Java class
|
58
58
|
# in this module's package
|
59
59
|
def resource_import(klass)
|
data/lib/caruby/import/java.rb
CHANGED
@@ -264,13 +264,13 @@ class Class
|
|
264
264
|
# * If the class argument is a Java class or a Java class name, then
|
265
265
|
# the Ruby class is the JRuby wrapper for the Java class.
|
266
266
|
#
|
267
|
-
# @param [Class, String] the class or class name
|
267
|
+
# @param [Class, String] class_or_name the class or class name
|
268
268
|
# @return [Class] the corresponding Ruby class
|
269
|
-
def self.to_ruby(
|
270
|
-
case
|
271
|
-
when Class then
|
272
|
-
when String then
|
273
|
-
else to_ruby(
|
269
|
+
def self.to_ruby(class_or_name)
|
270
|
+
case class_or_name
|
271
|
+
when Class then class_or_name
|
272
|
+
when String then eval to_ruby_name(class_or_name)
|
273
|
+
else to_ruby(class_or_name.name)
|
274
274
|
end
|
275
275
|
end
|
276
276
|
|
@@ -364,6 +364,18 @@ class Class
|
|
364
364
|
private
|
365
365
|
|
366
366
|
OBJ_INST_MTHDS = Object.instance_methods
|
367
|
+
|
368
|
+
# @param [String] jname the fully-qualified Java class or interface name
|
369
|
+
# @return [String] the JRuby class or module name
|
370
|
+
# #example
|
371
|
+
# Java.to_ruby_class_name('com.test.Sample') #=> Java::ComTest::Sample
|
372
|
+
def self.to_ruby_name(jname)
|
373
|
+
path = jname.split('.')
|
374
|
+
return "Java::#{jname}" if path.size == 1
|
375
|
+
cname = path[-1]
|
376
|
+
pkg = path[0...-1].map { |s| s.capitalize_first }.join
|
377
|
+
"Java::#{pkg}::#{cname}"
|
378
|
+
end
|
367
379
|
end
|
368
380
|
|
369
381
|
class Array
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module CaRuby
|
4
|
+
module JSON
|
5
|
+
# JSON => {Resource} deserializer.
|
6
|
+
module Deserializer
|
7
|
+
# @param [String] json the JSON to deserialize
|
8
|
+
# @return [Resource] the deserialized object
|
9
|
+
def json_create(json)
|
10
|
+
# Make the new object from the json data attribute => value hash.
|
11
|
+
new(json['data'])
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module CaRuby
|
2
|
+
module JSON
|
3
|
+
# {Resource} => JSON serializer.
|
4
|
+
module Serializer
|
5
|
+
# @param args the JSON serialization options
|
6
|
+
# @return [String] the JSON representation of this {Resource}
|
7
|
+
def to_json(*args)
|
8
|
+
database.lazy_loader.disable do
|
9
|
+
# The JSON class must be scoped by the domain module, not the Java package, in order
|
10
|
+
# to recognize the Resource JSON hooks.
|
11
|
+
# The data is the attribute => value hash.
|
12
|
+
{'json_class' => [self.class.domain_module, self.class.name.demodulize].join('::'),
|
13
|
+
'data' => value_hash(self.class.nonowner_attributes)
|
14
|
+
}.to_json(*args)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
module Enumerable
|
22
|
+
# @param args the JSON serialization options
|
23
|
+
# @return [String] the JSON representation of this Enumerable as an array
|
24
|
+
def to_json(*args)
|
25
|
+
to_a.to_json(*args)
|
26
|
+
end
|
27
|
+
end
|
@@ -84,16 +84,13 @@ module CaRuby
|
|
84
84
|
def migrate(row, migrated)
|
85
85
|
end
|
86
86
|
|
87
|
-
# Returns whether this migration target domain object is valid. The default is true
|
88
|
-
# if this domain object either has no owner or its owner is valid.
|
87
|
+
# Returns whether this migration target domain object is valid. The default is true.
|
89
88
|
# A migration shim should override this method on the target if there are conditions
|
90
89
|
# which determine whether the migration should be skipped for this target object.
|
91
90
|
#
|
92
91
|
# @return [Boolean] whether this migration target domain object is valid
|
93
92
|
def migration_valid?
|
94
|
-
|
95
|
-
ownr = owner
|
96
|
-
ownr.nil? or ownr.migration_valid?
|
93
|
+
true
|
97
94
|
end
|
98
95
|
|
99
96
|
# Migrates this domain object's migratable references. This method is called by the
|
@@ -115,7 +112,7 @@ module CaRuby
|
|
115
112
|
#
|
116
113
|
# @param [{Symbol => Object}] row the input row field => value hash
|
117
114
|
# @param [<Resource>] migrated the migrated instances, including this Resource
|
118
|
-
# @param [{Symbol => String}] mth_hash a hash that associates this domain object's
|
115
|
+
# @param [{Symbol => String}, nil] mth_hash a hash that associates this domain object's
|
119
116
|
# attributes to migration method names
|
120
117
|
def migrate_references(row, migrated, mth_hash=nil)
|
121
118
|
# migrate the owner
|
@@ -29,10 +29,11 @@ module CaRuby
|
|
29
29
|
# @param [{Symbol => Object}] opts the migration options
|
30
30
|
# @option opts [String] :database required application {Database}
|
31
31
|
# @option opts [String] :target required target domain class
|
32
|
-
# @option opts [String] :mapping required input field => caTissue attribute mapping file
|
33
|
-
# @option opts [String] :defaults optional caTissue attribute => value default mapping file
|
32
|
+
# @option opts [<String>, String] :mapping required input field => caTissue attribute mapping file(s)
|
33
|
+
# @option opts [<String>, String] :defaults optional caTissue attribute => value default mapping file(s)
|
34
|
+
# @option opts [<String>, String] :filters optional caTissue attribute input value => caTissue value filter file(s)
|
34
35
|
# @option opts [String] :input required source file to migrate
|
35
|
-
# @option opts [String] :shims optional
|
36
|
+
# @option opts [<String>, String] :shims optional shim file(s) to load
|
36
37
|
# @option opts [String] :unique ensures that migrated objects which include the {Resource::Unique}
|
37
38
|
# @option opts [String] :create optional flag indicating that existing target objects are ignored
|
38
39
|
# @option opts [String] :bad optional invalid record file
|
@@ -74,8 +75,8 @@ module CaRuby
|
|
74
75
|
alias :each :migrate
|
75
76
|
|
76
77
|
private
|
77
|
-
|
78
|
-
|
78
|
+
|
79
|
+
REGEXP_PAT = /^\/(.*[^\\])\/([inx]+)?$/
|
79
80
|
|
80
81
|
# Class {#migrate} with a {#save} block.
|
81
82
|
def execute_save
|
@@ -106,10 +107,10 @@ module CaRuby
|
|
106
107
|
end
|
107
108
|
|
108
109
|
def parse_options(opts)
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
@
|
110
|
+
@fld_map_files = opts[:mapping]
|
111
|
+
raise MigrationError.new("Migrator missing required field mapping file parameter") if @fld_map_files.nil?
|
112
|
+
@def_files = opts[:defaults]
|
113
|
+
@filter_files = opts[:filters]
|
113
114
|
@shims = opts[:shims] ||= []
|
114
115
|
@offset = opts[:offset] ||= 0
|
115
116
|
@input = Options.get(:input, opts)
|
@@ -136,14 +137,15 @@ module CaRuby
|
|
136
137
|
raise MigrationError.new("No file to migrate") if @input.nil?
|
137
138
|
|
138
139
|
# make a CSV loader which only converts input fields corresponding to non-String attributes
|
139
|
-
logger.info("Migration input file: #{@input}.")
|
140
140
|
@loader = CsvIO.new(@input, &method(:convert))
|
141
141
|
logger.debug { "Migration data input file #{@input} headers: #{@loader.headers.qp}" }
|
142
142
|
|
143
143
|
# create the class => path => default value hash
|
144
|
-
@def_hash = @
|
144
|
+
@def_hash = @def_files ? load_defaults_files(@def_files) : {}
|
145
|
+
# create the class => path => default value hash
|
146
|
+
@filter_hash = @filter_files ? load_filter_files(@filter_files) : {}
|
145
147
|
# create the class => path => header hash
|
146
|
-
fld_map =
|
148
|
+
fld_map = load_field_map_files(@fld_map_files)
|
147
149
|
# create the class => paths hash
|
148
150
|
@cls_paths_hash = create_class_paths_hash(fld_map, @def_hash)
|
149
151
|
# create the path => class => header hash
|
@@ -208,30 +210,40 @@ module CaRuby
|
|
208
210
|
|
209
211
|
# Adds missing klass owner classes to the migration class path hash (with empty paths).
|
210
212
|
def add_owners(klass)
|
211
|
-
klass
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
213
|
+
owner = missing_owner_for(klass) || return
|
214
|
+
logger.debug { "Migrator adding #{klass.qp} owner #{owner}" }
|
215
|
+
@cls_paths_hash[owner] = Array::EMPTY_ARRAY
|
216
|
+
add_owners(owner)
|
217
|
+
end
|
218
|
+
|
219
|
+
# @param [Class] klass the migration class
|
220
|
+
# @return [Class, nil] the missing class owner, if any
|
221
|
+
def missing_owner_for(klass)
|
222
|
+
# check for an owner among the current migration classes
|
223
|
+
return if klass.owners.any? do |owner|
|
224
|
+
@cls_paths_hash.detect_key { |other| other <= owner }
|
216
225
|
end
|
226
|
+
# find the first non-abstract candidate owner
|
227
|
+
klass.owners.detect { |owner| not owner.abstract? }
|
217
228
|
end
|
218
229
|
|
219
230
|
# Creates the class => +migrate_+_<attribute>_ hash for the given klasses.
|
220
231
|
def create_migration_method_hashes
|
221
|
-
# the attribute
|
222
|
-
@
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
232
|
+
# the class => attribute => migration filter hash
|
233
|
+
@attr_flt_hash = {}
|
234
|
+
customizable_class_attributes.each do |klass, attr_mds|
|
235
|
+
flts = migration_filters(klass, attr_mds) || next
|
236
|
+
@attr_flt_hash[klass] = flts
|
237
|
+
end
|
238
|
+
|
227
239
|
# print the migration shim methods
|
228
|
-
unless @
|
240
|
+
unless @attr_flt_hash.empty? then
|
229
241
|
printer_hash = LazyHash.new { Array.new }
|
230
|
-
@
|
231
|
-
|
232
|
-
printer_hash[klass.qp] =
|
242
|
+
@attr_flt_hash.each do |klass, hash|
|
243
|
+
mths = hash.values.select { |flt| Symbol === flt }
|
244
|
+
printer_hash[klass.qp] = mths unless mths.empty?
|
233
245
|
end
|
234
|
-
logger.info("Migration shim methods: #{printer_hash.pp_s}.")
|
246
|
+
logger.info("Migration shim methods: #{printer_hash.pp_s}.") unless printer_hash.empty?
|
235
247
|
end
|
236
248
|
end
|
237
249
|
|
@@ -275,13 +287,43 @@ module CaRuby
|
|
275
287
|
# Discovers methods of the form +migrate+__attribute_ implemented for the paths
|
276
288
|
# in the given class => paths hash the given klass. The migrate method is called
|
277
289
|
# on the input field value corresponding to the path.
|
278
|
-
def
|
290
|
+
def migration_filters(klass, attr_mds)
|
291
|
+
# the attribute => migration method hash
|
292
|
+
mth_hash = attribute_method_hash(klass, attr_mds)
|
293
|
+
proc_hash = attribute_proc_hash(klass, attr_mds)
|
294
|
+
return if mth_hash.empty? and proc_hash.empty?
|
295
|
+
|
296
|
+
# for each class path terminal attribute metadata, add the migration filters
|
297
|
+
# to the attribute metadata => filter hash
|
298
|
+
attr_mds.to_compact_hash do |attr_md|
|
299
|
+
# the filter proc
|
300
|
+
proc = proc_hash[attr_md.to_sym]
|
301
|
+
# the migration shim method
|
302
|
+
mth = mth_hash[attr_md.to_sym]
|
303
|
+
if mth then
|
304
|
+
if proc then
|
305
|
+
Proc.new do |obj, value, row|
|
306
|
+
# filter the value
|
307
|
+
fval = proc.call(value)
|
308
|
+
# call the migration method on the filtered value
|
309
|
+
obj.send(mth, fval, row) unless fval.nil?
|
310
|
+
end
|
311
|
+
else
|
312
|
+
# call the migration method
|
313
|
+
Proc.new { |obj, value, row| obj.send(mth, value, row) }
|
314
|
+
end
|
315
|
+
else
|
316
|
+
# call the filter
|
317
|
+
Proc.new { |obj, value, row| proc.call(value) }
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
def attribute_method_hash(klass, attr_mds)
|
279
323
|
# the migrate methods, excluding the Migratable migrate_references method
|
280
324
|
mths = klass.instance_methods(true).select { |mth| mth =~ /^migrate.(?!references)/ }
|
281
|
-
return if mths.empty?
|
282
|
-
|
283
325
|
# the attribute => migration method hash
|
284
|
-
|
326
|
+
mth_hash = {}
|
285
327
|
mths.each do |mth|
|
286
328
|
# the attribute suffix, e.g. name for migrate_name or Name for migrateName
|
287
329
|
suffix = /^migrate(_)?(.*)/.match(mth).captures[1]
|
@@ -290,24 +332,60 @@ module CaRuby
|
|
290
332
|
# the attribute for the name, or skip if no such attribute
|
291
333
|
attr = klass.standard_attribute(attr_nm) rescue next
|
292
334
|
# associate the attribute => method
|
293
|
-
|
335
|
+
mth_hash[attr] = mth
|
294
336
|
end
|
295
|
-
|
296
|
-
|
297
|
-
|
337
|
+
mth_hash
|
338
|
+
end
|
339
|
+
|
340
|
+
# @return [Attribute => {Object => Object}] the filter migration methods
|
341
|
+
def attribute_proc_hash(klass, attr_mds)
|
342
|
+
hash = @filter_hash[klass]
|
343
|
+
if hash.nil? then return Hash::EMPTY_HASH end
|
344
|
+
proc_hash = {}
|
298
345
|
attr_mds.each do |attr_md|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
346
|
+
flt = hash[attr_md.to_sym] || next
|
347
|
+
proc_hash[attr_md.to_sym] = to_filter_proc(flt)
|
348
|
+
end
|
349
|
+
logger.debug { "Migration filters loaded for #{klass.qp} #{proc_hash.keys.to_series}." }
|
350
|
+
proc_hash
|
351
|
+
end
|
352
|
+
|
353
|
+
# @param [{Object => Object}] filter the config value mapping
|
354
|
+
# @return [Proc] the filter migration block
|
355
|
+
def to_filter_proc(filter)
|
356
|
+
# Split the filter into a straight value => value hash and a regexp => value hash.
|
357
|
+
ph, vh = filter.split { |k, v| k =~ REGEXP_PAT }
|
358
|
+
reh = {}
|
359
|
+
ph.each do |k, v|
|
360
|
+
pat, opt = REGEXP_PAT.match(k).captures
|
361
|
+
reopt = if opt then
|
362
|
+
case opt
|
363
|
+
when 'i' then Regexp::IGNORECASE
|
364
|
+
else raise MigrationError.new("Migration value filter regular expression #{k} qualifier not supported: expected 'i', found '#{opt}'")
|
365
|
+
end
|
366
|
+
end
|
367
|
+
re = Regexp.new(pat, reopt)
|
368
|
+
reh[re] = v.gsub(/\$\d/, '%s') if String === v
|
369
|
+
end
|
370
|
+
Proc.new do |value|
|
371
|
+
if vh.has_key?(value) then
|
372
|
+
vh[value]
|
373
|
+
else
|
374
|
+
# the first regex which matches the value
|
375
|
+
regexp = reh.detect_key { |re| value =~ re }
|
376
|
+
# If there is a match, then apply the filter to the match data.
|
377
|
+
# Otherwise, pass the value through unmodified.
|
378
|
+
regexp ? (reh[regexp] % $~.captures) : value
|
379
|
+
end
|
303
380
|
end
|
304
|
-
@mgt_mth_hash[klass] = attr_mth_hash
|
305
381
|
end
|
306
382
|
|
307
|
-
#
|
383
|
+
# Loads the shim files.
|
384
|
+
#
|
385
|
+
# @param [<String>, String] files the file or file array
|
308
386
|
def load_shims(files)
|
309
387
|
logger.debug { "Loading shims with load path #{$:.pp_s}..." }
|
310
|
-
files.
|
388
|
+
files.enumerate do |file|
|
311
389
|
# load the file
|
312
390
|
begin
|
313
391
|
require file
|
@@ -323,7 +401,7 @@ module CaRuby
|
|
323
401
|
#
|
324
402
|
# @yield (see #migrate)
|
325
403
|
# @yieldparam (see #migrate)
|
326
|
-
def migrate_rows
|
404
|
+
def migrate_rows
|
327
405
|
# open an CSV output for bad records if the option is set
|
328
406
|
if @bad_rec_file then
|
329
407
|
@loader.trash = @bad_rec_file
|
@@ -359,7 +437,7 @@ module CaRuby
|
|
359
437
|
mgt_cnt += 1
|
360
438
|
# clear the migration state
|
361
439
|
clear(target)
|
362
|
-
|
440
|
+
else
|
363
441
|
# If there is a bad file then warn, reject and continue. Otherwise, bail.
|
364
442
|
if @bad_rec_file then
|
365
443
|
logger.warn("Migration not performed on record #{rec_no}.")
|
@@ -394,32 +472,55 @@ module CaRuby
|
|
394
472
|
#
|
395
473
|
# @param [{Symbol => Object}] row the input row field => value hash
|
396
474
|
# @return the migrated target object if the migration is valid, nil otherwise
|
397
|
-
def migrate_row(row)
|
475
|
+
def migrate_row(row)
|
398
476
|
# create an instance for each creatable class
|
399
477
|
created = Set.new
|
400
478
|
migrated = @creatable_classes.map { |klass| create(klass, row, created) }
|
401
479
|
# migrate each object from the input row
|
402
480
|
created.each { |obj| obj.migrate(row, migrated) }
|
403
|
-
|
404
|
-
valid, invalid = migrated.partition { |obj| migration_valid?(obj) }
|
405
|
-
# set the references
|
406
|
-
valid.each { |obj| obj.migrate_references(row, migrated, @mgt_mth_hash[obj.class]) }
|
481
|
+
valid = migrate_valid_references(row, migrated)
|
407
482
|
# the target object
|
408
483
|
target = valid.detect { |obj| @target_class === obj } || return
|
409
|
-
# the target is invalid if it has an invalid owner
|
410
|
-
return unless owner_valid?(target, valid, invalid)
|
411
484
|
logger.debug { "Migrated target #{target}." }
|
412
485
|
target
|
413
486
|
end
|
414
487
|
|
488
|
+
# Sets the migration references for each valid migrated object.
|
489
|
+
#
|
490
|
+
# @param [Array] the migrated objects
|
491
|
+
# @return [Array] the valid migrated objects
|
492
|
+
def migrate_valid_references(row, migrated)
|
493
|
+
# Split the valid and invalid objects. The iteration is in reverse dependency order,
|
494
|
+
# since invalidating a dependent can invalidate the owner.
|
495
|
+
valid, invalid = migrated.transitive_closure(:dependents).reverse.partition do |obj|
|
496
|
+
if migration_valid?(obj) then
|
497
|
+
obj.migrate_references(row, migrated, @attr_flt_hash[obj.class])
|
498
|
+
true
|
499
|
+
else
|
500
|
+
obj.class.owner_attributes.each { |attr| obj.clear_attribute(attr) }
|
501
|
+
false
|
502
|
+
end
|
503
|
+
end
|
504
|
+
|
505
|
+
# Go back through the valid objects in dependency order to invalidate dependents
|
506
|
+
# whose owner is invalid.
|
507
|
+
valid.reverse.each do |obj|
|
508
|
+
unless owner_valid?(obj, valid, invalid) then
|
509
|
+
invalid << valid.delete(obj)
|
510
|
+
logger.debug { "Invalidated migrated #{obj} since it does not have a valid owner." }
|
511
|
+
end
|
512
|
+
end
|
513
|
+
valid
|
514
|
+
end
|
515
|
+
|
415
516
|
# Returns whether the given domain object satisfies at least one of the following conditions:
|
416
|
-
# * it has an owner among the valid objects
|
417
517
|
# * it does not have an owner among the invalid objects
|
518
|
+
# * it has an owner among the valid objects
|
418
519
|
#
|
419
520
|
# @param [Resource] obj the domain object to check
|
420
521
|
# @param [<Resource>] valid the valid migrated objects
|
421
522
|
# @param [<Resource>] invalid the invalid migrated objects
|
422
|
-
# @return [Boolean] whether the
|
523
|
+
# @return [Boolean] whether the owner is valid
|
423
524
|
def owner_valid?(obj, valid, invalid)
|
424
525
|
otypes = obj.class.owners
|
425
526
|
invalid.all? { |other| not otypes.include?(other.class) } or
|
@@ -427,7 +528,7 @@ module CaRuby
|
|
427
528
|
end
|
428
529
|
|
429
530
|
# @param [Migratable] obj the migrated object
|
430
|
-
# @return whether the migration is successful
|
531
|
+
# @return [Boolean] whether the migration is successful
|
431
532
|
def migration_valid?(obj)
|
432
533
|
if obj.migration_valid? then
|
433
534
|
true
|
@@ -477,7 +578,8 @@ module CaRuby
|
|
477
578
|
# @param row (see #create)
|
478
579
|
# @param [<Resource>] created (see #create)
|
479
580
|
def add_defaults(obj, row, created)
|
480
|
-
@def_hash[obj.class]
|
581
|
+
dh = @def_hash[obj.class] || return
|
582
|
+
dh.each do |path, value|
|
481
583
|
# fill the reference path
|
482
584
|
ref = fill_path(obj, path[0...-1], row, created)
|
483
585
|
# set the attribute to the default value unless there is already a value
|
@@ -517,15 +619,19 @@ module CaRuby
|
|
517
619
|
ref
|
518
620
|
end
|
519
621
|
|
520
|
-
# Sets the
|
622
|
+
# Sets the given attribute value to the filtered input value. If there is a filter
|
623
|
+
# defined for the attribute, then that filter is applied. If there is a migration
|
624
|
+
# shim method with name +migrate_+_attribute_, then than method is called on the
|
625
|
+
# (possibly filtered) value. The target object attribute is set to the resulting
|
626
|
+
# filtered value.
|
627
|
+
#
|
628
|
+
# @param [Migratable] obj the target domain object
|
629
|
+
# @param [Attribute] attr_md the target attribute
|
630
|
+
# @param value the input value
|
631
|
+
# @param [{Symbol => Object}] row the input row
|
521
632
|
def migrate_attribute(obj, attr_md, value, row)
|
522
|
-
# a single value can be used for both a Numeric and a String attribute; coerce the value if necessary
|
523
633
|
# if there is a shim migrate_<attribute> method, then call it on the input value
|
524
|
-
|
525
|
-
if mth and obj.respond_to?(mth) then
|
526
|
-
value = obj.send(mth, value, row)
|
527
|
-
return if value.nil?
|
528
|
-
end
|
634
|
+
value = filter_value(obj, attr_md, value, row) || return
|
529
635
|
# set the attribute
|
530
636
|
begin
|
531
637
|
obj.send(attr_md.writer, value)
|
@@ -534,6 +640,25 @@ module CaRuby
|
|
534
640
|
end
|
535
641
|
logger.debug { "Migrated #{obj.qp} #{attr_md} to #{value}." }
|
536
642
|
end
|
643
|
+
|
644
|
+
# Calls the shim migrate_<attribute> method or config filter on the input value.
|
645
|
+
#
|
646
|
+
# @param value the input value
|
647
|
+
# @return the input value, if there is no filter, otherwise the filtered value
|
648
|
+
def filter_value(obj, attr_md, value, row)
|
649
|
+
filter = filter_for(obj, attr_md)
|
650
|
+
return value if filter.nil?
|
651
|
+
fval = filter.call(obj, value, row)
|
652
|
+
unless value == fval then
|
653
|
+
logger.debug { "Migration filter transformed #{obj.qp} #{attr_md} value from #{value.qp} to #{fval.qp}." }
|
654
|
+
end
|
655
|
+
fval
|
656
|
+
end
|
657
|
+
|
658
|
+
def filter_for(obj, attr_md)
|
659
|
+
flts = @attr_flt_hash[obj.class] || return
|
660
|
+
flts[attr_md]
|
661
|
+
end
|
537
662
|
|
538
663
|
# @param [Resource] obj the domain object to save in the database
|
539
664
|
# @return [Resource, nil] obj if the save is successful, nil otherwise
|
@@ -557,30 +682,12 @@ module CaRuby
|
|
557
682
|
@rec_cnt + 1
|
558
683
|
end
|
559
684
|
|
560
|
-
# @param [String]
|
685
|
+
# @param [<String>, String] files the migration fields mapping file or file array
|
561
686
|
# @return [{Class => {Attribute => Symbol}}] the class => path => header hash
|
562
|
-
# loaded from the
|
563
|
-
def
|
564
|
-
# load the field mapping config file
|
565
|
-
begin
|
566
|
-
config = YAML::load_file(file)
|
567
|
-
rescue
|
568
|
-
raise MigrationError.new("Could not read field map file #{file}: " + $!)
|
569
|
-
end
|
570
|
-
|
571
|
-
# collect the class => path => header entries
|
687
|
+
# loaded from the mapping files
|
688
|
+
def load_field_map_files(files)
|
572
689
|
map = LazyHash.new { Hash.new }
|
573
|
-
|
574
|
-
next if attr_list.blank?
|
575
|
-
# the header accessor method for the field
|
576
|
-
header = @loader.accessor(field)
|
577
|
-
raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
|
578
|
-
# associate each attribute path in the property value with the header
|
579
|
-
attr_list.split(/,\s*/).each do |path_s|
|
580
|
-
klass, path = create_attribute_path(path_s)
|
581
|
-
map[klass][path] = header
|
582
|
-
end
|
583
|
-
end
|
690
|
+
files.enumerate { |file| load_field_map_file(file, map) }
|
584
691
|
|
585
692
|
# include the target class
|
586
693
|
map[@target_class] ||= Hash.new
|
@@ -602,26 +709,95 @@ module CaRuby
|
|
602
709
|
map.delete_if do |klass, paths|
|
603
710
|
klass.abstract? or classes.any? { |other| other < klass }
|
604
711
|
end
|
712
|
+
|
605
713
|
map
|
606
714
|
end
|
607
715
|
|
608
|
-
|
716
|
+
# @param [String] file the migration fields configuration file
|
717
|
+
# @param [{Class => {Attribute => Symbol}}] hash the class => path => header hash
|
718
|
+
# loaded from the configuration file
|
719
|
+
def load_field_map_file(file, hash)
|
609
720
|
# load the field mapping config file
|
610
721
|
begin
|
611
722
|
config = YAML::load_file(file)
|
612
723
|
rescue
|
613
|
-
raise MigrationError.new("Could not read
|
724
|
+
raise MigrationError.new("Could not read field map file #{file}: " + $!)
|
614
725
|
end
|
615
726
|
|
727
|
+
# collect the class => path => header entries
|
728
|
+
config.each do |field, attr_list|
|
729
|
+
next if attr_list.blank?
|
730
|
+
# the header accessor method for the field
|
731
|
+
header = @loader.accessor(field)
|
732
|
+
raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
|
733
|
+
# associate each attribute path in the property value with the header
|
734
|
+
attr_list.split(/,\s*/).each do |path_s|
|
735
|
+
klass, path = create_attribute_path(path_s)
|
736
|
+
hash[klass][path] = header
|
737
|
+
end
|
738
|
+
end
|
739
|
+
end
|
740
|
+
|
741
|
+
# Loads the defaults config files.
|
742
|
+
#
|
743
|
+
# @param [<String>, String] files the file or file array to load
|
744
|
+
# @return [<Class => <String => Object>>] the class => path => default value entries
|
745
|
+
def load_defaults_files(files)
|
746
|
+
# collect the class => path => value entries from each defaults file
|
747
|
+
hash = LazyHash.new { Hash.new }
|
748
|
+
files.enumerate { |file| load_defaults_file(file, hash) }
|
749
|
+
hash
|
750
|
+
end
|
751
|
+
|
752
|
+
# Loads the defaults config file into the given hash.
|
753
|
+
#
|
754
|
+
# @param [String] file the file to load
|
755
|
+
# @param [<Class => <String => Object>>] hash the class => path => default value entries
|
756
|
+
def load_defaults_file(file, hash)
|
757
|
+
begin
|
758
|
+
config = YAML::load_file(file)
|
759
|
+
rescue
|
760
|
+
raise MigrationError.new("Could not read defaults file #{file}: " + $!)
|
761
|
+
end
|
616
762
|
# collect the class => path => value entries
|
617
|
-
map = LazyHash.new { Hash.new }
|
618
763
|
config.each do |path_s, value|
|
619
764
|
next if value.nil_or_empty?
|
620
765
|
klass, path = create_attribute_path(path_s)
|
621
|
-
|
766
|
+
hash[klass][path] = value
|
767
|
+
end
|
768
|
+
end
|
769
|
+
# Loads the filter config files.
|
770
|
+
#
|
771
|
+
# @param [<String>, String] files the file or file array to load
|
772
|
+
# @return [<Class => <String => Object>>] the class => path => default value entries
|
773
|
+
def load_filter_files(files)
|
774
|
+
# collect the class => path => value entries from each defaults file
|
775
|
+
hash = {}
|
776
|
+
files.enumerate { |file| load_filter_file(file, hash) }
|
777
|
+
hash
|
778
|
+
end
|
779
|
+
|
780
|
+
# Loads the filter config file into the given hash.
|
781
|
+
#
|
782
|
+
# @param [String] file the file to load
|
783
|
+
# @param [<Class => <String => <Object => Object>>>] hash the class => path => input value => caTissue value entries
|
784
|
+
def load_filter_file(file, hash)
|
785
|
+
begin
|
786
|
+
config = YAML::load_file(file)
|
787
|
+
rescue
|
788
|
+
raise MigrationError.new("Could not read filter file #{file}: " + $!)
|
789
|
+
end
|
790
|
+
# collect the class => attribute => filter entries
|
791
|
+
config.each do |path_s, flt|
|
792
|
+
next if flt.nil_or_empty?
|
793
|
+
klass, path = create_attribute_path(path_s)
|
794
|
+
unless path.size == 1 then
|
795
|
+
raise MigrationError.new("Migration filter configuration path not supported: #{path_s}")
|
796
|
+
end
|
797
|
+
attr = klass.standard_attribute(path.first.to_sym)
|
798
|
+
flt_hash = hash[klass] ||= {}
|
799
|
+
flt_hash[attr] = flt
|
622
800
|
end
|
623
|
-
|
624
|
-
map
|
625
801
|
end
|
626
802
|
|
627
803
|
# @param [String] path_s a period-delimited path string path_s in the form _class_(._attribute_)+
|