caruby-core 1.5.3 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.md +4 -0
- data/lib/caruby/cli/command.rb +2 -1
- data/lib/caruby/database/reader.rb +2 -3
- data/lib/caruby/database/search_template_builder.rb +5 -5
- data/lib/caruby/database/sql_executor.rb +7 -3
- data/lib/caruby/database/writer.rb +8 -9
- data/lib/caruby/domain/attribute.rb +6 -3
- data/lib/caruby/domain/attributes.rb +6 -1
- data/lib/caruby/domain/importer.rb +3 -3
- data/lib/caruby/import/java.rb +18 -6
- data/lib/caruby/json/deserializer.rb +15 -0
- data/lib/caruby/json/serializer.rb +27 -0
- data/lib/caruby/migration/migratable.rb +3 -6
- data/lib/caruby/migration/migrator.rb +267 -91
- data/lib/caruby/resource.rb +5 -3
- data/lib/caruby/util/collection.rb +8 -15
- data/lib/caruby/util/log.rb +1 -1
- data/lib/caruby/util/properties.rb +1 -1
- data/lib/caruby/util/topological_sync_enumerator.rb +4 -1
- data/lib/caruby/util/transitive_closure.rb +19 -9
- data/lib/caruby/version.rb +1 -1
- data/test/lib/caruby/util/collection_test.rb +2 -6
- data/test/lib/caruby/util/transitive_closure_test.rb +21 -13
- data/test/lib/examples/clinical_trials/migration/migration_test.rb +58 -0
- data/test/lib/examples/clinical_trials/migration/test_case.rb +38 -0
- metadata +8 -6
- data/test/lib/examples/galena/clinical_trials/migration/participant_test.rb +0 -28
- data/test/lib/examples/galena/clinical_trials/migration/test_case.rb +0 -40
data/History.md
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
This history lists major release themes. See the GitHub Commits (https://github.com/caruby/core)
|
2
2
|
for change details.
|
3
3
|
|
4
|
+
1.5.4 / 2011-07-19
|
5
|
+
------------------
|
6
|
+
* Add migration value filter option.
|
7
|
+
|
4
8
|
1.5.3 / 2011-07-08
|
5
9
|
------------------
|
6
10
|
* Clean up documentation.
|
data/lib/caruby/cli/command.rb
CHANGED
@@ -32,11 +32,12 @@ module CaRuby
|
|
32
32
|
#
|
33
33
|
# Built-in options include the following:
|
34
34
|
# * --help : print the help message and exit
|
35
|
-
# * --
|
35
|
+
# * --verbose : print additional information to the console
|
36
36
|
# * --log FILE : log file
|
37
37
|
# * --debug : print debug messages to the log
|
38
38
|
# * --file FILE: configuration file containing other options
|
39
39
|
# * --quiet: suppress printing messages to stdout
|
40
|
+
#
|
40
41
|
# This class processes these built-in options, with the exception of +--version+,
|
41
42
|
# which is a subclass responsibility. Subclasses are responsible for
|
42
43
|
# processing any remaining options.
|
@@ -321,15 +321,14 @@ module CaRuby
|
|
321
321
|
return
|
322
322
|
end
|
323
323
|
@transients << obj
|
324
|
-
|
325
324
|
logger.debug { "Fetching #{obj.qp} from the database..." }
|
326
325
|
fetched = fetch_object(obj) || return
|
327
|
-
# fetch_object can return obj; if so, then done
|
326
|
+
# fetch_object can return obj; if so, then done, otherwise, merge fetched.
|
328
327
|
return obj if obj.equal?(fetched)
|
329
328
|
|
330
329
|
logger.debug { "Fetch #{obj.qp} matched database object #{fetched}." }
|
331
330
|
@transients.delete(obj)
|
332
|
-
#
|
331
|
+
# recursively copy the nondomain attributes, esp. the identifer, of the fetched domain object references
|
333
332
|
merge_fetched(fetched, obj)
|
334
333
|
# Inject the lazy loader for loadable domain reference attributes.
|
335
334
|
persistify(obj, fetched)
|
@@ -24,14 +24,14 @@ module CaRuby
|
|
24
24
|
logger.debug { "Building search template for #{obj.qp}..." }
|
25
25
|
hash ||= obj.value_hash(obj.class.searchable_attributes)
|
26
26
|
# the searchable attribute => value hash
|
27
|
-
|
27
|
+
rh, nrh = hash.split { |attr, value| Resource === value }
|
28
28
|
# make the search template from the non-reference attributes
|
29
|
-
tmpl = obj.class.new.merge_attributes(
|
29
|
+
tmpl = obj.class.new.merge_attributes(nrh)
|
30
30
|
# get references for the search template
|
31
|
-
unless
|
32
|
-
logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{
|
31
|
+
unless rh.empty? then
|
32
|
+
logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{rh.keys.to_series}..." }
|
33
33
|
end
|
34
|
-
|
34
|
+
rh.each { |attr, ref| add_search_template_reference(tmpl, ref, attr) }
|
35
35
|
tmpl
|
36
36
|
end
|
37
37
|
|
@@ -34,6 +34,9 @@ module CaRuby
|
|
34
34
|
# @option opts [String] :database_driver_class the optional DBI connect driver class name
|
35
35
|
# @raise [CaRuby::ConfigurationError] if an option is invalid
|
36
36
|
def initialize(opts)
|
37
|
+
if opts.empty? then
|
38
|
+
raise CaRuby::ConfigurationError.new("The caRuby database connection properties were not found.")
|
39
|
+
end
|
37
40
|
app_host = Options.get(:host, opts, 'localhost')
|
38
41
|
db_host = Options.get(:database_host, opts, app_host)
|
39
42
|
db_type = Options.get(:database_type, opts, 'mysql')
|
@@ -52,9 +55,10 @@ module CaRuby
|
|
52
55
|
:database_type => db_type,
|
53
56
|
:database_port => db_port,
|
54
57
|
:database_driver => db_driver,
|
55
|
-
:database_driver_class => @driver_class
|
56
|
-
|
57
|
-
|
58
|
+
:database_driver_class => @driver_class,
|
59
|
+
:address => @address
|
60
|
+
}
|
61
|
+
logger.debug { "Database connection parameters (excluding password): #{eff_opts.qp}" }
|
58
62
|
end
|
59
63
|
|
60
64
|
# Connects to the database, yields the DBI handle to the given block and disconnects.
|
@@ -139,6 +139,7 @@ module CaRuby
|
|
139
139
|
def ensure_exists(obj)
|
140
140
|
raise ArgumentError.new("Database ensure_exists is missing a domain object argument") if obj.nil_or_empty?
|
141
141
|
obj.enumerate { |ref| find(ref, :create) unless ref.identifier }
|
142
|
+
|
142
143
|
end
|
143
144
|
|
144
145
|
private
|
@@ -286,7 +287,6 @@ module CaRuby
|
|
286
287
|
logger.debug { "Ensuring that created #{obj.qp} references exist: #{refs.qp}..." } unless refs.empty?
|
287
288
|
refs.each { |ref| ensure_exists(ref) }
|
288
289
|
end
|
289
|
-
|
290
290
|
obj
|
291
291
|
end
|
292
292
|
|
@@ -350,7 +350,7 @@ module CaRuby
|
|
350
350
|
# * the attribute references a {#pending_create?} save context.
|
351
351
|
#
|
352
352
|
# @param obj (see #create)
|
353
|
-
# @param [Attribute] attr_md candidate attribute metadata
|
353
|
+
# @param [Attribute] attr_md the candidate attribute metadata
|
354
354
|
# @return [Boolean] whether the attribute should not be included in the create template
|
355
355
|
def exclude_pending_create_attribute?(obj, attr_md)
|
356
356
|
attr_md.independent? and
|
@@ -413,7 +413,7 @@ module CaRuby
|
|
413
413
|
# update a cascaded dependent by updating the owner
|
414
414
|
owner = cascaded_owner(obj)
|
415
415
|
if owner then return update_cascaded_dependent(owner, obj) end
|
416
|
-
|
416
|
+
|
417
417
|
# Not cascaded dependent; update using a template,
|
418
418
|
tmpl = build_update_template(obj)
|
419
419
|
# call the caCORE service with an obj update template
|
@@ -559,10 +559,9 @@ module CaRuby
|
|
559
559
|
logger.debug { "Updating saved #{target} to store unsaved attributes..." }
|
560
560
|
# call update_object(saved) rather than update(saved) to bypass the redundant update check
|
561
561
|
perform(:update, target) { update_object(target) }
|
562
|
-
else
|
563
|
-
# recursively save the dependents
|
564
|
-
save_changed_dependents(target)
|
565
562
|
end
|
563
|
+
# recursively save the dependents
|
564
|
+
save_changed_dependents(target)
|
566
565
|
end
|
567
566
|
|
568
567
|
# Synchronizes the given saved target result source with the database content.
|
@@ -685,9 +684,9 @@ module CaRuby
|
|
685
684
|
# @param [Resource] obj the owner domain object
|
686
685
|
def save_changed_dependents(obj)
|
687
686
|
obj.class.dependent_attributes.each do |attr|
|
688
|
-
deps = obj.send(attr)
|
689
|
-
logger.debug { "Saving the #{obj} #{attr} dependents #{deps.qp(:single_line)} which have changed..." } unless deps.
|
690
|
-
deps.
|
687
|
+
deps = obj.send(attr)
|
688
|
+
logger.debug { "Saving the #{obj} #{attr} dependents #{deps.to_enum.qp(:single_line)} which have changed..." } unless deps.nil_or_empty?
|
689
|
+
deps.enumerate { |dep| save_dependent_if_changed(obj, attr, dep) }
|
691
690
|
end
|
692
691
|
end
|
693
692
|
|
@@ -127,7 +127,7 @@ module CaRuby
|
|
127
127
|
# The inverse relation is symmetric, i.e. the inverse of the referenced Attribute
|
128
128
|
# is set to this Attribute's subject attribute.
|
129
129
|
#
|
130
|
-
# @param attribute the inverse attribute
|
130
|
+
# @param [Symbol, nil] attribute the inverse attribute
|
131
131
|
def inverse=(attribute)
|
132
132
|
return if inverse == attribute
|
133
133
|
# if no attribute, then the clear the existing inverse, if any
|
@@ -522,9 +522,12 @@ module CaRuby
|
|
522
522
|
def clear_inverse
|
523
523
|
return unless @inv_md
|
524
524
|
logger.debug { "Clearing #{@declarer.qp}.#{self} inverse #{type.qp}.#{inverse}..." }
|
525
|
-
|
525
|
+
# Capture the inverse before unsetting it.
|
526
|
+
inv_md = @inv_md
|
527
|
+
# Unset the inverse.
|
526
528
|
@inv_md = nil
|
527
|
-
|
529
|
+
# Clear the inverse of the inverse.
|
530
|
+
inv_md.inverse = nil
|
528
531
|
logger.debug { "Cleared #{@declarer.qp}.#{self} inverse." }
|
529
532
|
end
|
530
533
|
|
@@ -530,8 +530,13 @@ module CaRuby
|
|
530
530
|
def remove_attribute(attribute)
|
531
531
|
std_attr = standard_attribute(attribute)
|
532
532
|
# if the attribute is local, then delete it, otherwise filter out the superclass attribute
|
533
|
-
|
533
|
+
attr_md = @local_attr_md_hash.delete(std_attr)
|
534
|
+
if attr_md then
|
535
|
+
# clear the inverse, if any
|
536
|
+
attr_md.inverse = nil
|
537
|
+
# remove from the mandatory attributes, if necessary
|
534
538
|
@local_mndty_attrs.delete(std_attr)
|
539
|
+
# remove from the attribute => metadata hash
|
535
540
|
@local_std_attr_hash.delete_if { |aliaz, attr| attr == std_attr }
|
536
541
|
else
|
537
542
|
# Filter the superclass hashes.
|
@@ -39,8 +39,8 @@ module CaRuby
|
|
39
39
|
logger.debug { "Detecting whether #{symbol} is a #{@pkg} Java class..." }
|
40
40
|
# Append the symbol to the package to make the Java class name.
|
41
41
|
begin
|
42
|
-
klass =
|
43
|
-
resource_import
|
42
|
+
klass = java_import "#{@pkg}.#{symbol}"
|
43
|
+
resource_import(klass)
|
44
44
|
rescue NameError
|
45
45
|
logger.debug { "#{symbol} is not recognized as a #{@pkg} Java class - #{$!}\n#{caller.qp}." }
|
46
46
|
super
|
@@ -53,7 +53,7 @@ module CaRuby
|
|
53
53
|
# The Java class is assumed to be defined in this module's package.
|
54
54
|
# This module's mixin is added to the class.
|
55
55
|
#
|
56
|
-
# @param [
|
56
|
+
# @param [Class] klass the source directory
|
57
57
|
# @raise [NameError] if the symbol does not correspond to a Java class
|
58
58
|
# in this module's package
|
59
59
|
def resource_import(klass)
|
data/lib/caruby/import/java.rb
CHANGED
@@ -264,13 +264,13 @@ class Class
|
|
264
264
|
# * If the class argument is a Java class or a Java class name, then
|
265
265
|
# the Ruby class is the JRuby wrapper for the Java class.
|
266
266
|
#
|
267
|
-
# @param [Class, String] the class or class name
|
267
|
+
# @param [Class, String] class_or_name the class or class name
|
268
268
|
# @return [Class] the corresponding Ruby class
|
269
|
-
def self.to_ruby(
|
270
|
-
case
|
271
|
-
when Class then
|
272
|
-
when String then
|
273
|
-
else to_ruby(
|
269
|
+
def self.to_ruby(class_or_name)
|
270
|
+
case class_or_name
|
271
|
+
when Class then class_or_name
|
272
|
+
when String then eval to_ruby_name(class_or_name)
|
273
|
+
else to_ruby(class_or_name.name)
|
274
274
|
end
|
275
275
|
end
|
276
276
|
|
@@ -364,6 +364,18 @@ class Class
|
|
364
364
|
private
|
365
365
|
|
366
366
|
OBJ_INST_MTHDS = Object.instance_methods
|
367
|
+
|
368
|
+
# @param [String] jname the fully-qualified Java class or interface name
|
369
|
+
# @return [String] the JRuby class or module name
|
370
|
+
# #example
|
371
|
+
# Java.to_ruby_class_name('com.test.Sample') #=> Java::ComTest::Sample
|
372
|
+
def self.to_ruby_name(jname)
|
373
|
+
path = jname.split('.')
|
374
|
+
return "Java::#{jname}" if path.size == 1
|
375
|
+
cname = path[-1]
|
376
|
+
pkg = path[0...-1].map { |s| s.capitalize_first }.join
|
377
|
+
"Java::#{pkg}::#{cname}"
|
378
|
+
end
|
367
379
|
end
|
368
380
|
|
369
381
|
class Array
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module CaRuby
|
4
|
+
module JSON
|
5
|
+
# JSON => {Resource} deserializer.
|
6
|
+
module Deserializer
|
7
|
+
# @param [String] json the JSON to deserialize
|
8
|
+
# @return [Resource] the deserialized object
|
9
|
+
def json_create(json)
|
10
|
+
# Make the new object from the json data attribute => value hash.
|
11
|
+
new(json['data'])
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module CaRuby
|
2
|
+
module JSON
|
3
|
+
# {Resource} => JSON serializer.
|
4
|
+
module Serializer
|
5
|
+
# @param args the JSON serialization options
|
6
|
+
# @return [String] the JSON representation of this {Resource}
|
7
|
+
def to_json(*args)
|
8
|
+
database.lazy_loader.disable do
|
9
|
+
# The JSON class must be scoped by the domain module, not the Java package, in order
|
10
|
+
# to recognize the Resource JSON hooks.
|
11
|
+
# The data is the attribute => value hash.
|
12
|
+
{'json_class' => [self.class.domain_module, self.class.name.demodulize].join('::'),
|
13
|
+
'data' => value_hash(self.class.nonowner_attributes)
|
14
|
+
}.to_json(*args)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
module Enumerable
|
22
|
+
# @param args the JSON serialization options
|
23
|
+
# @return [String] the JSON representation of this Enumerable as an array
|
24
|
+
def to_json(*args)
|
25
|
+
to_a.to_json(*args)
|
26
|
+
end
|
27
|
+
end
|
@@ -84,16 +84,13 @@ module CaRuby
|
|
84
84
|
def migrate(row, migrated)
|
85
85
|
end
|
86
86
|
|
87
|
-
# Returns whether this migration target domain object is valid. The default is true
|
88
|
-
# if this domain object either has no owner or its owner is valid.
|
87
|
+
# Returns whether this migration target domain object is valid. The default is true.
|
89
88
|
# A migration shim should override this method on the target if there are conditions
|
90
89
|
# which determine whether the migration should be skipped for this target object.
|
91
90
|
#
|
92
91
|
# @return [Boolean] whether this migration target domain object is valid
|
93
92
|
def migration_valid?
|
94
|
-
|
95
|
-
ownr = owner
|
96
|
-
ownr.nil? or ownr.migration_valid?
|
93
|
+
true
|
97
94
|
end
|
98
95
|
|
99
96
|
# Migrates this domain object's migratable references. This method is called by the
|
@@ -115,7 +112,7 @@ module CaRuby
|
|
115
112
|
#
|
116
113
|
# @param [{Symbol => Object}] row the input row field => value hash
|
117
114
|
# @param [<Resource>] migrated the migrated instances, including this Resource
|
118
|
-
# @param [{Symbol => String}] mth_hash a hash that associates this domain object's
|
115
|
+
# @param [{Symbol => String}, nil] mth_hash a hash that associates this domain object's
|
119
116
|
# attributes to migration method names
|
120
117
|
def migrate_references(row, migrated, mth_hash=nil)
|
121
118
|
# migrate the owner
|
@@ -29,10 +29,11 @@ module CaRuby
|
|
29
29
|
# @param [{Symbol => Object}] opts the migration options
|
30
30
|
# @option opts [String] :database required application {Database}
|
31
31
|
# @option opts [String] :target required target domain class
|
32
|
-
# @option opts [String] :mapping required input field => caTissue attribute mapping file
|
33
|
-
# @option opts [String] :defaults optional caTissue attribute => value default mapping file
|
32
|
+
# @option opts [<String>, String] :mapping required input field => caTissue attribute mapping file(s)
|
33
|
+
# @option opts [<String>, String] :defaults optional caTissue attribute => value default mapping file(s)
|
34
|
+
# @option opts [<String>, String] :filters optional caTissue attribute input value => caTissue value filter file(s)
|
34
35
|
# @option opts [String] :input required source file to migrate
|
35
|
-
# @option opts [String] :shims optional
|
36
|
+
# @option opts [<String>, String] :shims optional shim file(s) to load
|
36
37
|
# @option opts [String] :unique ensures that migrated objects which include the {Resource::Unique}
|
37
38
|
# @option opts [String] :create optional flag indicating that existing target objects are ignored
|
38
39
|
# @option opts [String] :bad optional invalid record file
|
@@ -74,8 +75,8 @@ module CaRuby
|
|
74
75
|
alias :each :migrate
|
75
76
|
|
76
77
|
private
|
77
|
-
|
78
|
-
|
78
|
+
|
79
|
+
REGEXP_PAT = /^\/(.*[^\\])\/([inx]+)?$/
|
79
80
|
|
80
81
|
# Class {#migrate} with a {#save} block.
|
81
82
|
def execute_save
|
@@ -106,10 +107,10 @@ module CaRuby
|
|
106
107
|
end
|
107
108
|
|
108
109
|
def parse_options(opts)
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
@
|
110
|
+
@fld_map_files = opts[:mapping]
|
111
|
+
raise MigrationError.new("Migrator missing required field mapping file parameter") if @fld_map_files.nil?
|
112
|
+
@def_files = opts[:defaults]
|
113
|
+
@filter_files = opts[:filters]
|
113
114
|
@shims = opts[:shims] ||= []
|
114
115
|
@offset = opts[:offset] ||= 0
|
115
116
|
@input = Options.get(:input, opts)
|
@@ -136,14 +137,15 @@ module CaRuby
|
|
136
137
|
raise MigrationError.new("No file to migrate") if @input.nil?
|
137
138
|
|
138
139
|
# make a CSV loader which only converts input fields corresponding to non-String attributes
|
139
|
-
logger.info("Migration input file: #{@input}.")
|
140
140
|
@loader = CsvIO.new(@input, &method(:convert))
|
141
141
|
logger.debug { "Migration data input file #{@input} headers: #{@loader.headers.qp}" }
|
142
142
|
|
143
143
|
# create the class => path => default value hash
|
144
|
-
@def_hash = @
|
144
|
+
@def_hash = @def_files ? load_defaults_files(@def_files) : {}
|
145
|
+
# create the class => path => default value hash
|
146
|
+
@filter_hash = @filter_files ? load_filter_files(@filter_files) : {}
|
145
147
|
# create the class => path => header hash
|
146
|
-
fld_map =
|
148
|
+
fld_map = load_field_map_files(@fld_map_files)
|
147
149
|
# create the class => paths hash
|
148
150
|
@cls_paths_hash = create_class_paths_hash(fld_map, @def_hash)
|
149
151
|
# create the path => class => header hash
|
@@ -208,30 +210,40 @@ module CaRuby
|
|
208
210
|
|
209
211
|
# Adds missing klass owner classes to the migration class path hash (with empty paths).
|
210
212
|
def add_owners(klass)
|
211
|
-
klass
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
213
|
+
owner = missing_owner_for(klass) || return
|
214
|
+
logger.debug { "Migrator adding #{klass.qp} owner #{owner}" }
|
215
|
+
@cls_paths_hash[owner] = Array::EMPTY_ARRAY
|
216
|
+
add_owners(owner)
|
217
|
+
end
|
218
|
+
|
219
|
+
# @param [Class] klass the migration class
|
220
|
+
# @return [Class, nil] the missing class owner, if any
|
221
|
+
def missing_owner_for(klass)
|
222
|
+
# check for an owner among the current migration classes
|
223
|
+
return if klass.owners.any? do |owner|
|
224
|
+
@cls_paths_hash.detect_key { |other| other <= owner }
|
216
225
|
end
|
226
|
+
# find the first non-abstract candidate owner
|
227
|
+
klass.owners.detect { |owner| not owner.abstract? }
|
217
228
|
end
|
218
229
|
|
219
230
|
# Creates the class => +migrate_+_<attribute>_ hash for the given klasses.
|
220
231
|
def create_migration_method_hashes
|
221
|
-
# the attribute
|
222
|
-
@
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
232
|
+
# the class => attribute => migration filter hash
|
233
|
+
@attr_flt_hash = {}
|
234
|
+
customizable_class_attributes.each do |klass, attr_mds|
|
235
|
+
flts = migration_filters(klass, attr_mds) || next
|
236
|
+
@attr_flt_hash[klass] = flts
|
237
|
+
end
|
238
|
+
|
227
239
|
# print the migration shim methods
|
228
|
-
unless @
|
240
|
+
unless @attr_flt_hash.empty? then
|
229
241
|
printer_hash = LazyHash.new { Array.new }
|
230
|
-
@
|
231
|
-
|
232
|
-
printer_hash[klass.qp] =
|
242
|
+
@attr_flt_hash.each do |klass, hash|
|
243
|
+
mths = hash.values.select { |flt| Symbol === flt }
|
244
|
+
printer_hash[klass.qp] = mths unless mths.empty?
|
233
245
|
end
|
234
|
-
logger.info("Migration shim methods: #{printer_hash.pp_s}.")
|
246
|
+
logger.info("Migration shim methods: #{printer_hash.pp_s}.") unless printer_hash.empty?
|
235
247
|
end
|
236
248
|
end
|
237
249
|
|
@@ -275,13 +287,43 @@ module CaRuby
|
|
275
287
|
# Discovers methods of the form +migrate+__attribute_ implemented for the paths
|
276
288
|
# in the given class => paths hash the given klass. The migrate method is called
|
277
289
|
# on the input field value corresponding to the path.
|
278
|
-
def
|
290
|
+
def migration_filters(klass, attr_mds)
|
291
|
+
# the attribute => migration method hash
|
292
|
+
mth_hash = attribute_method_hash(klass, attr_mds)
|
293
|
+
proc_hash = attribute_proc_hash(klass, attr_mds)
|
294
|
+
return if mth_hash.empty? and proc_hash.empty?
|
295
|
+
|
296
|
+
# for each class path terminal attribute metadata, add the migration filters
|
297
|
+
# to the attribute metadata => filter hash
|
298
|
+
attr_mds.to_compact_hash do |attr_md|
|
299
|
+
# the filter proc
|
300
|
+
proc = proc_hash[attr_md.to_sym]
|
301
|
+
# the migration shim method
|
302
|
+
mth = mth_hash[attr_md.to_sym]
|
303
|
+
if mth then
|
304
|
+
if proc then
|
305
|
+
Proc.new do |obj, value, row|
|
306
|
+
# filter the value
|
307
|
+
fval = proc.call(value)
|
308
|
+
# call the migration method on the filtered value
|
309
|
+
obj.send(mth, fval, row) unless fval.nil?
|
310
|
+
end
|
311
|
+
else
|
312
|
+
# call the migration method
|
313
|
+
Proc.new { |obj, value, row| obj.send(mth, value, row) }
|
314
|
+
end
|
315
|
+
else
|
316
|
+
# call the filter
|
317
|
+
Proc.new { |obj, value, row| proc.call(value) }
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
def attribute_method_hash(klass, attr_mds)
|
279
323
|
# the migrate methods, excluding the Migratable migrate_references method
|
280
324
|
mths = klass.instance_methods(true).select { |mth| mth =~ /^migrate.(?!references)/ }
|
281
|
-
return if mths.empty?
|
282
|
-
|
283
325
|
# the attribute => migration method hash
|
284
|
-
|
326
|
+
mth_hash = {}
|
285
327
|
mths.each do |mth|
|
286
328
|
# the attribute suffix, e.g. name for migrate_name or Name for migrateName
|
287
329
|
suffix = /^migrate(_)?(.*)/.match(mth).captures[1]
|
@@ -290,24 +332,60 @@ module CaRuby
|
|
290
332
|
# the attribute for the name, or skip if no such attribute
|
291
333
|
attr = klass.standard_attribute(attr_nm) rescue next
|
292
334
|
# associate the attribute => method
|
293
|
-
|
335
|
+
mth_hash[attr] = mth
|
294
336
|
end
|
295
|
-
|
296
|
-
|
297
|
-
|
337
|
+
mth_hash
|
338
|
+
end
|
339
|
+
|
340
|
+
# @return [Attribute => {Object => Object}] the filter migration methods
|
341
|
+
def attribute_proc_hash(klass, attr_mds)
|
342
|
+
hash = @filter_hash[klass]
|
343
|
+
if hash.nil? then return Hash::EMPTY_HASH end
|
344
|
+
proc_hash = {}
|
298
345
|
attr_mds.each do |attr_md|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
346
|
+
flt = hash[attr_md.to_sym] || next
|
347
|
+
proc_hash[attr_md.to_sym] = to_filter_proc(flt)
|
348
|
+
end
|
349
|
+
logger.debug { "Migration filters loaded for #{klass.qp} #{proc_hash.keys.to_series}." }
|
350
|
+
proc_hash
|
351
|
+
end
|
352
|
+
|
353
|
+
# @param [{Object => Object}] filter the config value mapping
|
354
|
+
# @return [Proc] the filter migration block
|
355
|
+
def to_filter_proc(filter)
|
356
|
+
# Split the filter into a straight value => value hash and a regexp => value hash.
|
357
|
+
ph, vh = filter.split { |k, v| k =~ REGEXP_PAT }
|
358
|
+
reh = {}
|
359
|
+
ph.each do |k, v|
|
360
|
+
pat, opt = REGEXP_PAT.match(k).captures
|
361
|
+
reopt = if opt then
|
362
|
+
case opt
|
363
|
+
when 'i' then Regexp::IGNORECASE
|
364
|
+
else raise MigrationError.new("Migration value filter regular expression #{k} qualifier not supported: expected 'i', found '#{opt}'")
|
365
|
+
end
|
366
|
+
end
|
367
|
+
re = Regexp.new(pat, reopt)
|
368
|
+
reh[re] = v.gsub(/\$\d/, '%s') if String === v
|
369
|
+
end
|
370
|
+
Proc.new do |value|
|
371
|
+
if vh.has_key?(value) then
|
372
|
+
vh[value]
|
373
|
+
else
|
374
|
+
# the first regex which matches the value
|
375
|
+
regexp = reh.detect_key { |re| value =~ re }
|
376
|
+
# If there is a match, then apply the filter to the match data.
|
377
|
+
# Otherwise, pass the value through unmodified.
|
378
|
+
regexp ? (reh[regexp] % $~.captures) : value
|
379
|
+
end
|
303
380
|
end
|
304
|
-
@mgt_mth_hash[klass] = attr_mth_hash
|
305
381
|
end
|
306
382
|
|
307
|
-
#
|
383
|
+
# Loads the shim files.
|
384
|
+
#
|
385
|
+
# @param [<String>, String] files the file or file array
|
308
386
|
def load_shims(files)
|
309
387
|
logger.debug { "Loading shims with load path #{$:.pp_s}..." }
|
310
|
-
files.
|
388
|
+
files.enumerate do |file|
|
311
389
|
# load the file
|
312
390
|
begin
|
313
391
|
require file
|
@@ -323,7 +401,7 @@ module CaRuby
|
|
323
401
|
#
|
324
402
|
# @yield (see #migrate)
|
325
403
|
# @yieldparam (see #migrate)
|
326
|
-
def migrate_rows
|
404
|
+
def migrate_rows
|
327
405
|
# open an CSV output for bad records if the option is set
|
328
406
|
if @bad_rec_file then
|
329
407
|
@loader.trash = @bad_rec_file
|
@@ -359,7 +437,7 @@ module CaRuby
|
|
359
437
|
mgt_cnt += 1
|
360
438
|
# clear the migration state
|
361
439
|
clear(target)
|
362
|
-
|
440
|
+
else
|
363
441
|
# If there is a bad file then warn, reject and continue. Otherwise, bail.
|
364
442
|
if @bad_rec_file then
|
365
443
|
logger.warn("Migration not performed on record #{rec_no}.")
|
@@ -394,32 +472,55 @@ module CaRuby
|
|
394
472
|
#
|
395
473
|
# @param [{Symbol => Object}] row the input row field => value hash
|
396
474
|
# @return the migrated target object if the migration is valid, nil otherwise
|
397
|
-
def migrate_row(row)
|
475
|
+
def migrate_row(row)
|
398
476
|
# create an instance for each creatable class
|
399
477
|
created = Set.new
|
400
478
|
migrated = @creatable_classes.map { |klass| create(klass, row, created) }
|
401
479
|
# migrate each object from the input row
|
402
480
|
created.each { |obj| obj.migrate(row, migrated) }
|
403
|
-
|
404
|
-
valid, invalid = migrated.partition { |obj| migration_valid?(obj) }
|
405
|
-
# set the references
|
406
|
-
valid.each { |obj| obj.migrate_references(row, migrated, @mgt_mth_hash[obj.class]) }
|
481
|
+
valid = migrate_valid_references(row, migrated)
|
407
482
|
# the target object
|
408
483
|
target = valid.detect { |obj| @target_class === obj } || return
|
409
|
-
# the target is invalid if it has an invalid owner
|
410
|
-
return unless owner_valid?(target, valid, invalid)
|
411
484
|
logger.debug { "Migrated target #{target}." }
|
412
485
|
target
|
413
486
|
end
|
414
487
|
|
488
|
+
# Sets the migration references for each valid migrated object.
|
489
|
+
#
|
490
|
+
# @param [Array] the migrated objects
|
491
|
+
# @return [Array] the valid migrated objects
|
492
|
+
def migrate_valid_references(row, migrated)
|
493
|
+
# Split the valid and invalid objects. The iteration is in reverse dependency order,
|
494
|
+
# since invalidating a dependent can invalidate the owner.
|
495
|
+
valid, invalid = migrated.transitive_closure(:dependents).reverse.partition do |obj|
|
496
|
+
if migration_valid?(obj) then
|
497
|
+
obj.migrate_references(row, migrated, @attr_flt_hash[obj.class])
|
498
|
+
true
|
499
|
+
else
|
500
|
+
obj.class.owner_attributes.each { |attr| obj.clear_attribute(attr) }
|
501
|
+
false
|
502
|
+
end
|
503
|
+
end
|
504
|
+
|
505
|
+
# Go back through the valid objects in dependency order to invalidate dependents
|
506
|
+
# whose owner is invalid.
|
507
|
+
valid.reverse.each do |obj|
|
508
|
+
unless owner_valid?(obj, valid, invalid) then
|
509
|
+
invalid << valid.delete(obj)
|
510
|
+
logger.debug { "Invalidated migrated #{obj} since it does not have a valid owner." }
|
511
|
+
end
|
512
|
+
end
|
513
|
+
valid
|
514
|
+
end
|
515
|
+
|
415
516
|
# Returns whether the given domain object satisfies at least one of the following conditions:
|
416
|
-
# * it has an owner among the valid objects
|
417
517
|
# * it does not have an owner among the invalid objects
|
518
|
+
# * it has an owner among the valid objects
|
418
519
|
#
|
419
520
|
# @param [Resource] obj the domain object to check
|
420
521
|
# @param [<Resource>] valid the valid migrated objects
|
421
522
|
# @param [<Resource>] invalid the invalid migrated objects
|
422
|
-
# @return [Boolean] whether the
|
523
|
+
# @return [Boolean] whether the owner is valid
|
423
524
|
def owner_valid?(obj, valid, invalid)
|
424
525
|
otypes = obj.class.owners
|
425
526
|
invalid.all? { |other| not otypes.include?(other.class) } or
|
@@ -427,7 +528,7 @@ module CaRuby
|
|
427
528
|
end
|
428
529
|
|
429
530
|
# @param [Migratable] obj the migrated object
|
430
|
-
# @return whether the migration is successful
|
531
|
+
# @return [Boolean] whether the migration is successful
|
431
532
|
def migration_valid?(obj)
|
432
533
|
if obj.migration_valid? then
|
433
534
|
true
|
@@ -477,7 +578,8 @@ module CaRuby
|
|
477
578
|
# @param row (see #create)
|
478
579
|
# @param [<Resource>] created (see #create)
|
479
580
|
def add_defaults(obj, row, created)
|
480
|
-
@def_hash[obj.class]
|
581
|
+
dh = @def_hash[obj.class] || return
|
582
|
+
dh.each do |path, value|
|
481
583
|
# fill the reference path
|
482
584
|
ref = fill_path(obj, path[0...-1], row, created)
|
483
585
|
# set the attribute to the default value unless there is already a value
|
@@ -517,15 +619,19 @@ module CaRuby
|
|
517
619
|
ref
|
518
620
|
end
|
519
621
|
|
520
|
-
# Sets the
|
622
|
+
# Sets the given attribute value to the filtered input value. If there is a filter
|
623
|
+
# defined for the attribute, then that filter is applied. If there is a migration
|
624
|
+
# shim method with name +migrate_+_attribute_, then than method is called on the
|
625
|
+
# (possibly filtered) value. The target object attribute is set to the resulting
|
626
|
+
# filtered value.
|
627
|
+
#
|
628
|
+
# @param [Migratable] obj the target domain object
|
629
|
+
# @param [Attribute] attr_md the target attribute
|
630
|
+
# @param value the input value
|
631
|
+
# @param [{Symbol => Object}] row the input row
|
521
632
|
def migrate_attribute(obj, attr_md, value, row)
|
522
|
-
# a single value can be used for both a Numeric and a String attribute; coerce the value if necessary
|
523
633
|
# if there is a shim migrate_<attribute> method, then call it on the input value
|
524
|
-
|
525
|
-
if mth and obj.respond_to?(mth) then
|
526
|
-
value = obj.send(mth, value, row)
|
527
|
-
return if value.nil?
|
528
|
-
end
|
634
|
+
value = filter_value(obj, attr_md, value, row) || return
|
529
635
|
# set the attribute
|
530
636
|
begin
|
531
637
|
obj.send(attr_md.writer, value)
|
@@ -534,6 +640,25 @@ module CaRuby
|
|
534
640
|
end
|
535
641
|
logger.debug { "Migrated #{obj.qp} #{attr_md} to #{value}." }
|
536
642
|
end
|
643
|
+
|
644
|
+
# Calls the shim migrate_<attribute> method or config filter on the input value.
|
645
|
+
#
|
646
|
+
# @param value the input value
|
647
|
+
# @return the input value, if there is no filter, otherwise the filtered value
|
648
|
+
def filter_value(obj, attr_md, value, row)
|
649
|
+
filter = filter_for(obj, attr_md)
|
650
|
+
return value if filter.nil?
|
651
|
+
fval = filter.call(obj, value, row)
|
652
|
+
unless value == fval then
|
653
|
+
logger.debug { "Migration filter transformed #{obj.qp} #{attr_md} value from #{value.qp} to #{fval.qp}." }
|
654
|
+
end
|
655
|
+
fval
|
656
|
+
end
|
657
|
+
|
658
|
+
def filter_for(obj, attr_md)
|
659
|
+
flts = @attr_flt_hash[obj.class] || return
|
660
|
+
flts[attr_md]
|
661
|
+
end
|
537
662
|
|
538
663
|
# @param [Resource] obj the domain object to save in the database
|
539
664
|
# @return [Resource, nil] obj if the save is successful, nil otherwise
|
@@ -557,30 +682,12 @@ module CaRuby
|
|
557
682
|
@rec_cnt + 1
|
558
683
|
end
|
559
684
|
|
560
|
-
# @param [String]
|
685
|
+
# @param [<String>, String] files the migration fields mapping file or file array
|
561
686
|
# @return [{Class => {Attribute => Symbol}}] the class => path => header hash
|
562
|
-
# loaded from the
|
563
|
-
def
|
564
|
-
# load the field mapping config file
|
565
|
-
begin
|
566
|
-
config = YAML::load_file(file)
|
567
|
-
rescue
|
568
|
-
raise MigrationError.new("Could not read field map file #{file}: " + $!)
|
569
|
-
end
|
570
|
-
|
571
|
-
# collect the class => path => header entries
|
687
|
+
# loaded from the mapping files
|
688
|
+
def load_field_map_files(files)
|
572
689
|
map = LazyHash.new { Hash.new }
|
573
|
-
|
574
|
-
next if attr_list.blank?
|
575
|
-
# the header accessor method for the field
|
576
|
-
header = @loader.accessor(field)
|
577
|
-
raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
|
578
|
-
# associate each attribute path in the property value with the header
|
579
|
-
attr_list.split(/,\s*/).each do |path_s|
|
580
|
-
klass, path = create_attribute_path(path_s)
|
581
|
-
map[klass][path] = header
|
582
|
-
end
|
583
|
-
end
|
690
|
+
files.enumerate { |file| load_field_map_file(file, map) }
|
584
691
|
|
585
692
|
# include the target class
|
586
693
|
map[@target_class] ||= Hash.new
|
@@ -602,26 +709,95 @@ module CaRuby
|
|
602
709
|
map.delete_if do |klass, paths|
|
603
710
|
klass.abstract? or classes.any? { |other| other < klass }
|
604
711
|
end
|
712
|
+
|
605
713
|
map
|
606
714
|
end
|
607
715
|
|
608
|
-
|
716
|
+
# @param [String] file the migration fields configuration file
|
717
|
+
# @param [{Class => {Attribute => Symbol}}] hash the class => path => header hash
|
718
|
+
# loaded from the configuration file
|
719
|
+
def load_field_map_file(file, hash)
|
609
720
|
# load the field mapping config file
|
610
721
|
begin
|
611
722
|
config = YAML::load_file(file)
|
612
723
|
rescue
|
613
|
-
raise MigrationError.new("Could not read
|
724
|
+
raise MigrationError.new("Could not read field map file #{file}: " + $!)
|
614
725
|
end
|
615
726
|
|
727
|
+
# collect the class => path => header entries
|
728
|
+
config.each do |field, attr_list|
|
729
|
+
next if attr_list.blank?
|
730
|
+
# the header accessor method for the field
|
731
|
+
header = @loader.accessor(field)
|
732
|
+
raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
|
733
|
+
# associate each attribute path in the property value with the header
|
734
|
+
attr_list.split(/,\s*/).each do |path_s|
|
735
|
+
klass, path = create_attribute_path(path_s)
|
736
|
+
hash[klass][path] = header
|
737
|
+
end
|
738
|
+
end
|
739
|
+
end
|
740
|
+
|
741
|
+
# Loads the defaults config files.
|
742
|
+
#
|
743
|
+
# @param [<String>, String] files the file or file array to load
|
744
|
+
# @return [<Class => <String => Object>>] the class => path => default value entries
|
745
|
+
def load_defaults_files(files)
|
746
|
+
# collect the class => path => value entries from each defaults file
|
747
|
+
hash = LazyHash.new { Hash.new }
|
748
|
+
files.enumerate { |file| load_defaults_file(file, hash) }
|
749
|
+
hash
|
750
|
+
end
|
751
|
+
|
752
|
+
# Loads the defaults config file into the given hash.
|
753
|
+
#
|
754
|
+
# @param [String] file the file to load
|
755
|
+
# @param [<Class => <String => Object>>] hash the class => path => default value entries
|
756
|
+
def load_defaults_file(file, hash)
|
757
|
+
begin
|
758
|
+
config = YAML::load_file(file)
|
759
|
+
rescue
|
760
|
+
raise MigrationError.new("Could not read defaults file #{file}: " + $!)
|
761
|
+
end
|
616
762
|
# collect the class => path => value entries
|
617
|
-
map = LazyHash.new { Hash.new }
|
618
763
|
config.each do |path_s, value|
|
619
764
|
next if value.nil_or_empty?
|
620
765
|
klass, path = create_attribute_path(path_s)
|
621
|
-
|
766
|
+
hash[klass][path] = value
|
767
|
+
end
|
768
|
+
end
|
769
|
+
# Loads the filter config files.
|
770
|
+
#
|
771
|
+
# @param [<String>, String] files the file or file array to load
|
772
|
+
# @return [<Class => <String => Object>>] the class => path => default value entries
|
773
|
+
def load_filter_files(files)
|
774
|
+
# collect the class => path => value entries from each defaults file
|
775
|
+
hash = {}
|
776
|
+
files.enumerate { |file| load_filter_file(file, hash) }
|
777
|
+
hash
|
778
|
+
end
|
779
|
+
|
780
|
+
# Loads the filter config file into the given hash.
|
781
|
+
#
|
782
|
+
# @param [String] file the file to load
|
783
|
+
# @param [<Class => <String => <Object => Object>>>] hash the class => path => input value => caTissue value entries
|
784
|
+
def load_filter_file(file, hash)
|
785
|
+
begin
|
786
|
+
config = YAML::load_file(file)
|
787
|
+
rescue
|
788
|
+
raise MigrationError.new("Could not read filter file #{file}: " + $!)
|
789
|
+
end
|
790
|
+
# collect the class => attribute => filter entries
|
791
|
+
config.each do |path_s, flt|
|
792
|
+
next if flt.nil_or_empty?
|
793
|
+
klass, path = create_attribute_path(path_s)
|
794
|
+
unless path.size == 1 then
|
795
|
+
raise MigrationError.new("Migration filter configuration path not supported: #{path_s}")
|
796
|
+
end
|
797
|
+
attr = klass.standard_attribute(path.first.to_sym)
|
798
|
+
flt_hash = hash[klass] ||= {}
|
799
|
+
flt_hash[attr] = flt
|
622
800
|
end
|
623
|
-
|
624
|
-
map
|
625
801
|
end
|
626
802
|
|
627
803
|
# @param [String] path_s a period-delimited path string path_s in the form _class_(._attribute_)+
|