ncs_mdes_warehouse 0.14.0 → 0.15.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +9 -0
- data/lib/ncs_navigator/warehouse/cli.rb +12 -1
- data/lib/ncs_navigator/warehouse/configuration.rb +3 -1
- data/lib/ncs_navigator/warehouse/configuration/file_evaluator.rb +1 -1
- data/lib/ncs_navigator/warehouse/database_initializer.rb +20 -1
- data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb +6 -5
- data/lib/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer.rb +15 -15
- data/lib/ncs_navigator/warehouse/transformers/subprocess_transformer.rb +1 -0
- data/lib/ncs_navigator/warehouse/version.rb +1 -1
- data/ncs_mdes_warehouse.gemspec +1 -1
- data/spec/ncs_navigator/warehouse/database_initializer_spec.rb +27 -0
- metadata +8 -11
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,15 @@
|
|
1
1
|
NCS Navigator MDES Warehouse History
|
2
2
|
====================================
|
3
3
|
|
4
|
+
0.15.0
|
5
|
+
------
|
6
|
+
|
7
|
+
- Add soft_validations option: Record DataMapper validations
|
8
|
+
(as transform errors) but still attempt to export (#4315)
|
9
|
+
|
10
|
+
- Add drop_not_null option: Drop NOT NULL constraints for all, but
|
11
|
+
the primary key colums in the database (#4316)
|
12
|
+
|
4
13
|
0.14.0
|
5
14
|
------
|
6
15
|
|
@@ -4,10 +4,18 @@ require 'thor'
|
|
4
4
|
|
5
5
|
module NcsNavigator::Warehouse
|
6
6
|
class CLI < Thor
|
7
|
+
|
8
|
+
def initialize(args=[], options={}, config={})
|
9
|
+
super
|
10
|
+
@options = @options.dup
|
11
|
+
end
|
12
|
+
|
7
13
|
class_option :quiet, :type => :boolean, :aliases => %w(-q),
|
8
14
|
:desc => 'Suppress the status messages printed to standard error'
|
9
15
|
class_option 'config', :type => :string, :aliases => %w(-c),
|
10
16
|
:desc => "Supply an alternate configuration file instead of the default #{Configuration.environment_file}"
|
17
|
+
class_option :soft_validations, :type => :boolean,
|
18
|
+
:desc => "Record DataMapper validations (as transform errors) but still attempt to export"
|
11
19
|
|
12
20
|
no_tasks {
|
13
21
|
def configuration
|
@@ -21,6 +29,7 @@ module NcsNavigator::Warehouse
|
|
21
29
|
c.output_level = options['quiet'] ? :quiet : :normal
|
22
30
|
end
|
23
31
|
c.set_up_logs
|
32
|
+
c.soft_validations = options['soft_validations']
|
24
33
|
end
|
25
34
|
end
|
26
35
|
end
|
@@ -89,10 +98,13 @@ DESC
|
|
89
98
|
:desc => 'Copy the working schema to production even if there are errors'
|
90
99
|
method_option 'preserve', :type => 'boolean',
|
91
100
|
:desc => 'Do not wipe the working database before beginning ETL (for debugging)'
|
101
|
+
method_option 'drop_not_null', :type => 'boolean',
|
102
|
+
:desc => 'Drop NOT NULL constraints for all but the primary key columns in the database'
|
92
103
|
def etl
|
93
104
|
db = DatabaseInitializer.new(configuration)
|
94
105
|
db.set_up_repository(:both)
|
95
106
|
db.replace_schema unless options[:preserve]
|
107
|
+
db.drop_all_null_constraints(:working) if options[:drop_not_null]
|
96
108
|
|
97
109
|
success = TransformLoad.new(configuration).run
|
98
110
|
if success || options['force']
|
@@ -174,4 +186,3 @@ DESC
|
|
174
186
|
end
|
175
187
|
end
|
176
188
|
end
|
177
|
-
|
@@ -406,6 +406,9 @@ module NcsNavigator::Warehouse
|
|
406
406
|
@output_level ||= :normal
|
407
407
|
end
|
408
408
|
|
409
|
+
# try to save records with DataMapper validation errors i.e. record.valid?
|
410
|
+
attr_accessor :soft_validations
|
411
|
+
|
409
412
|
##
|
410
413
|
# Set the desired terminal output level.
|
411
414
|
#
|
@@ -560,4 +563,3 @@ module NcsNavigator::Warehouse
|
|
560
563
|
end
|
561
564
|
end
|
562
565
|
end
|
563
|
-
|
@@ -93,6 +93,26 @@ module NcsNavigator::Warehouse
|
|
93
93
|
create_no_pii_views
|
94
94
|
end
|
95
95
|
|
96
|
+
def drop_all_null_constraints(which, options={})
|
97
|
+
shell.clear_line_then_say "Dropping all NOT NULL constraints in #{which} schema" unless options[:quiet]
|
98
|
+
models = ::DataMapper::Model.descendants
|
99
|
+
adapter = ::DataMapper.repository(:"mdes_warehouse_#{which}").adapter
|
100
|
+
|
101
|
+
models.each do |model|
|
102
|
+
model.properties.each do |property|
|
103
|
+
# Skip if NULLS are allowed or column is part of the primary key
|
104
|
+
next if property.allow_blank? || model.key.include?(property)
|
105
|
+
adapter.execute(%Q|
|
106
|
+
ALTER TABLE #{model.storage_name}
|
107
|
+
ALTER COLUMN #{property.name}
|
108
|
+
DROP NOT NULL
|
109
|
+
|)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
shell.clear_line_then_say "Dropped all NOT NULL constraints in #{which} schema.\n" unless options[:quiet]
|
114
|
+
end
|
115
|
+
|
96
116
|
# @private Exposed for use in tests
|
97
117
|
def drop_all(which, options={})
|
98
118
|
shell.clear_line_then_say "Dropping everything in #{which} schema" unless options[:quiet]
|
@@ -252,4 +272,3 @@ module NcsNavigator::Warehouse
|
|
252
272
|
private :escape_cmd
|
253
273
|
end
|
254
274
|
end
|
255
|
-
|
@@ -66,7 +66,7 @@ module NcsNavigator::Warehouse::Transformers
|
|
66
66
|
@duplicates_strategy = select_duplicates_strategy
|
67
67
|
|
68
68
|
@record_checkers = {
|
69
|
-
:validation => ValidateRecordChecker.new(log),
|
69
|
+
:validation => ValidateRecordChecker.new(log, @configuration),
|
70
70
|
:foreign_key => ForeignKeyChecker.new(log, foreign_key_index),
|
71
71
|
:psus => PsuIdChecker.new(log, @configuration.navigator.psus)
|
72
72
|
}
|
@@ -139,11 +139,11 @@ module NcsNavigator::Warehouse::Transformers
|
|
139
139
|
if saveable
|
140
140
|
log.debug("Saving verified record #{record_ident record}.")
|
141
141
|
begin
|
142
|
-
if record.save
|
142
|
+
if (@configuration.soft_validations ? record.save! : record.save)
|
143
143
|
record
|
144
144
|
foreign_key_index.record(record)
|
145
145
|
else
|
146
|
-
msg = "Could not save
|
146
|
+
msg = "Could not save record #{record.inspect}.\n\t Errors: #{record.errors.inspect}"
|
147
147
|
log.error msg
|
148
148
|
status.unsuccessful_record(record, msg)
|
149
149
|
end
|
@@ -225,8 +225,9 @@ module NcsNavigator::Warehouse::Transformers
|
|
225
225
|
|
226
226
|
attr_reader :log
|
227
227
|
|
228
|
-
def initialize(log)
|
228
|
+
def initialize(log, configuration)
|
229
229
|
@log = log
|
230
|
+
@configuration = configuration
|
230
231
|
end
|
231
232
|
|
232
233
|
def verify_or_report_errors(record, status)
|
@@ -244,7 +245,7 @@ module NcsNavigator::Warehouse::Transformers
|
|
244
245
|
)
|
245
246
|
end
|
246
247
|
end
|
247
|
-
|
248
|
+
@configuration.soft_validations
|
248
249
|
end
|
249
250
|
end
|
250
251
|
|
@@ -50,12 +50,24 @@ module NcsNavigator::Warehouse::Transformers
|
|
50
50
|
private
|
51
51
|
|
52
52
|
# TODO: share this stuff between here and EnumTransformer
|
53
|
-
|
54
53
|
def save(record, status)
|
55
|
-
if record.valid?
|
54
|
+
if !(valid = record.valid?)
|
55
|
+
log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
|
56
|
+
record.errors.keys.each do |prop|
|
57
|
+
record.errors[prop].each do |e|
|
58
|
+
status.unsuccessful_record(
|
59
|
+
record, "Invalid after transformation: #{e}.",
|
60
|
+
:attribute_name => prop,
|
61
|
+
:attribute_value => record.send(prop).inspect
|
62
|
+
)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
if valid || @configuration.soft_validations
|
56
68
|
log.debug("Saving valid transformed event record #{record_ident record}.")
|
57
69
|
begin
|
58
|
-
unless record.save
|
70
|
+
unless (@configuration.soft_validations ? record.save! : record.save)
|
59
71
|
msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
|
60
72
|
log.error msg
|
61
73
|
status.unsuccessful_record(record, msg)
|
@@ -65,17 +77,6 @@ module NcsNavigator::Warehouse::Transformers
|
|
65
77
|
log.error msg
|
66
78
|
status.unsuccessful_record(record, msg)
|
67
79
|
end
|
68
|
-
else
|
69
|
-
log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
|
70
|
-
record.errors.keys.each do |prop|
|
71
|
-
record.errors[prop].each do |e|
|
72
|
-
status.unsuccessful_record(
|
73
|
-
record, "Invalid after transformation: #{e}.",
|
74
|
-
:attribute_name => prop,
|
75
|
-
:attribute_value => record.send(prop).inspect
|
76
|
-
)
|
77
|
-
end
|
78
|
-
end
|
79
80
|
end
|
80
81
|
end
|
81
82
|
|
@@ -95,4 +96,3 @@ module NcsNavigator::Warehouse::Transformers
|
|
95
96
|
end
|
96
97
|
end
|
97
98
|
end
|
98
|
-
|
@@ -68,6 +68,7 @@ module NcsNavigator::Warehouse::Transformers
|
|
68
68
|
def initialize(config, exec_and_args, options={})
|
69
69
|
@configuration = config
|
70
70
|
@exec_and_args = exec_and_args
|
71
|
+
@exec_and_args << '--soft_validations' if @configuration.soft_validations
|
71
72
|
@directory = options[:directory] || '.'
|
72
73
|
@environment = options[:environment] || {}
|
73
74
|
end
|
data/ncs_mdes_warehouse.gemspec
CHANGED
@@ -48,5 +48,5 @@ Gem::Specification.new do |s|
|
|
48
48
|
s.add_development_dependency 'rake', '~> 0.9.2'
|
49
49
|
s.add_development_dependency 'yard', '~> 0.7.2'
|
50
50
|
s.add_development_dependency 'ci_reporter', '1.6.6'
|
51
|
-
s.add_development_dependency 'fakefs', '0.4.
|
51
|
+
s.add_development_dependency 'fakefs', '0.4.2' # FakeFS does not follow semver
|
52
52
|
end
|
@@ -2,6 +2,33 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
module NcsNavigator::Warehouse
|
4
4
|
describe DatabaseInitializer do
|
5
|
+
describe '#drop_all_null_constraints', :slow do
|
6
|
+
it 'drops NOT NULL constraints on all non-primary key columns in all tables' do
|
7
|
+
subject = DatabaseInitializer.new(spec_config)
|
8
|
+
subject.set_up_repository(:working)
|
9
|
+
subject.replace_schema
|
10
|
+
subject.drop_all_null_constraints(:working)
|
11
|
+
|
12
|
+
models = ::DataMapper::Model.descendants
|
13
|
+
adapter = ::DataMapper.repository(:mdes_warehouse_working).adapter
|
14
|
+
|
15
|
+
models.each do |m|
|
16
|
+
nullables = adapter.select(%Q{
|
17
|
+
SELECT cs.column_name
|
18
|
+
FROM information_schema.columns AS cs
|
19
|
+
WHERE cs.is_nullable = 'YES'
|
20
|
+
AND cs.table_name = '#{m.storage_name}'
|
21
|
+
})
|
22
|
+
|
23
|
+
m.properties.each do |p|
|
24
|
+
unless m.key.include?(p)
|
25
|
+
nullables.should include(p.name.to_s)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
5
32
|
describe '#replace_schema', :slow, :use_mdes do
|
6
33
|
subject { DatabaseInitializer.new(spec_config) }
|
7
34
|
let(:mdes_models) { spec_config.models_module.mdes_order }
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ncs_mdes_warehouse
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.15.0.pre1
|
5
|
+
prerelease: 7
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Rhett Sutphin
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-06-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ncs_mdes
|
@@ -452,7 +452,7 @@ dependencies:
|
|
452
452
|
requirements:
|
453
453
|
- - '='
|
454
454
|
- !ruby/object:Gem::Version
|
455
|
-
version: 0.4.
|
455
|
+
version: 0.4.2
|
456
456
|
type: :development
|
457
457
|
prerelease: false
|
458
458
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -460,7 +460,7 @@ dependencies:
|
|
460
460
|
requirements:
|
461
461
|
- - '='
|
462
462
|
- !ruby/object:Gem::Version
|
463
|
-
version: 0.4.
|
463
|
+
version: 0.4.2
|
464
464
|
description:
|
465
465
|
email:
|
466
466
|
- r-sutphin@northwestern.edu
|
@@ -3669,16 +3669,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
3669
3669
|
version: '0'
|
3670
3670
|
segments:
|
3671
3671
|
- 0
|
3672
|
-
hash: -
|
3672
|
+
hash: -4565215284799470018
|
3673
3673
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
3674
3674
|
none: false
|
3675
3675
|
requirements:
|
3676
|
-
- - ! '
|
3676
|
+
- - ! '>'
|
3677
3677
|
- !ruby/object:Gem::Version
|
3678
|
-
version:
|
3679
|
-
segments:
|
3680
|
-
- 0
|
3681
|
-
hash: -3845842505256535190
|
3678
|
+
version: 1.3.1
|
3682
3679
|
requirements: []
|
3683
3680
|
rubyforge_project:
|
3684
3681
|
rubygems_version: 1.8.25
|