ncs_mdes_warehouse 0.14.0 → 0.15.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG.md CHANGED
@@ -1,6 +1,15 @@
1
1
  NCS Navigator MDES Warehouse History
2
2
  ====================================
3
3
 
4
+ 0.15.0
5
+ ------
6
+
7
+ - Add soft_validations option: Record DataMapper validations
8
+ (as transform errors) but still attempt to export (#4315)
9
+
10
+ - Add drop_not_null option: Drop NOT NULL constraints for all, but
11
+ the primary key colums in the database (#4316)
12
+
4
13
  0.14.0
5
14
  ------
6
15
 
@@ -4,10 +4,18 @@ require 'thor'
4
4
 
5
5
  module NcsNavigator::Warehouse
6
6
  class CLI < Thor
7
+
8
+ def initialize(args=[], options={}, config={})
9
+ super
10
+ @options = @options.dup
11
+ end
12
+
7
13
  class_option :quiet, :type => :boolean, :aliases => %w(-q),
8
14
  :desc => 'Suppress the status messages printed to standard error'
9
15
  class_option 'config', :type => :string, :aliases => %w(-c),
10
16
  :desc => "Supply an alternate configuration file instead of the default #{Configuration.environment_file}"
17
+ class_option :soft_validations, :type => :boolean,
18
+ :desc => "Record DataMapper validations (as transform errors) but still attempt to export"
11
19
 
12
20
  no_tasks {
13
21
  def configuration
@@ -21,6 +29,7 @@ module NcsNavigator::Warehouse
21
29
  c.output_level = options['quiet'] ? :quiet : :normal
22
30
  end
23
31
  c.set_up_logs
32
+ c.soft_validations = options['soft_validations']
24
33
  end
25
34
  end
26
35
  end
@@ -89,10 +98,13 @@ DESC
89
98
  :desc => 'Copy the working schema to production even if there are errors'
90
99
  method_option 'preserve', :type => 'boolean',
91
100
  :desc => 'Do not wipe the working database before beginning ETL (for debugging)'
101
+ method_option 'drop_not_null', :type => 'boolean',
102
+ :desc => 'Drop NOT NULL constraints for all but the primary key columns in the database'
92
103
  def etl
93
104
  db = DatabaseInitializer.new(configuration)
94
105
  db.set_up_repository(:both)
95
106
  db.replace_schema unless options[:preserve]
107
+ db.drop_all_null_constraints(:working) if options[:drop_not_null]
96
108
 
97
109
  success = TransformLoad.new(configuration).run
98
110
  if success || options['force']
@@ -174,4 +186,3 @@ DESC
174
186
  end
175
187
  end
176
188
  end
177
-
@@ -406,6 +406,9 @@ module NcsNavigator::Warehouse
406
406
  @output_level ||= :normal
407
407
  end
408
408
 
409
+ # try to save records with DataMapper validation errors i.e. record.valid?
410
+ attr_accessor :soft_validations
411
+
409
412
  ##
410
413
  # Set the desired terminal output level.
411
414
  #
@@ -560,4 +563,3 @@ module NcsNavigator::Warehouse
560
563
  end
561
564
  end
562
565
  end
563
-
@@ -17,7 +17,7 @@ class NcsNavigator::Warehouse::Configuration
17
17
  end
18
18
 
19
19
  def eval
20
- instance_eval(File.read(@filename), @filename)
20
+ instance_eval(File.read(@filename.to_s), @filename.to_s)
21
21
  configuration.finish
22
22
  @evaled = true
23
23
  end
@@ -93,6 +93,26 @@ module NcsNavigator::Warehouse
93
93
  create_no_pii_views
94
94
  end
95
95
 
96
+ def drop_all_null_constraints(which, options={})
97
+ shell.clear_line_then_say "Dropping all NOT NULL constraints in #{which} schema" unless options[:quiet]
98
+ models = ::DataMapper::Model.descendants
99
+ adapter = ::DataMapper.repository(:"mdes_warehouse_#{which}").adapter
100
+
101
+ models.each do |model|
102
+ model.properties.each do |property|
103
+ # Skip if NULLS are allowed or column is part of the primary key
104
+ next if property.allow_blank? || model.key.include?(property)
105
+ adapter.execute(%Q|
106
+ ALTER TABLE #{model.storage_name}
107
+ ALTER COLUMN #{property.name}
108
+ DROP NOT NULL
109
+ |)
110
+ end
111
+ end
112
+
113
+ shell.clear_line_then_say "Dropped all NOT NULL constraints in #{which} schema.\n" unless options[:quiet]
114
+ end
115
+
96
116
  # @private Exposed for use in tests
97
117
  def drop_all(which, options={})
98
118
  shell.clear_line_then_say "Dropping everything in #{which} schema" unless options[:quiet]
@@ -252,4 +272,3 @@ module NcsNavigator::Warehouse
252
272
  private :escape_cmd
253
273
  end
254
274
  end
255
-
@@ -66,7 +66,7 @@ module NcsNavigator::Warehouse::Transformers
66
66
  @duplicates_strategy = select_duplicates_strategy
67
67
 
68
68
  @record_checkers = {
69
- :validation => ValidateRecordChecker.new(log),
69
+ :validation => ValidateRecordChecker.new(log, @configuration),
70
70
  :foreign_key => ForeignKeyChecker.new(log, foreign_key_index),
71
71
  :psus => PsuIdChecker.new(log, @configuration.navigator.psus)
72
72
  }
@@ -139,11 +139,11 @@ module NcsNavigator::Warehouse::Transformers
139
139
  if saveable
140
140
  log.debug("Saving verified record #{record_ident record}.")
141
141
  begin
142
- if record.save
142
+ if (@configuration.soft_validations ? record.save! : record.save)
143
143
  record
144
144
  foreign_key_index.record(record)
145
145
  else
146
- msg = "Could not save valid record #{record.inspect}."
146
+ msg = "Could not save record #{record.inspect}.\n\t Errors: #{record.errors.inspect}"
147
147
  log.error msg
148
148
  status.unsuccessful_record(record, msg)
149
149
  end
@@ -225,8 +225,9 @@ module NcsNavigator::Warehouse::Transformers
225
225
 
226
226
  attr_reader :log
227
227
 
228
- def initialize(log)
228
+ def initialize(log, configuration)
229
229
  @log = log
230
+ @configuration = configuration
230
231
  end
231
232
 
232
233
  def verify_or_report_errors(record, status)
@@ -244,7 +245,7 @@ module NcsNavigator::Warehouse::Transformers
244
245
  )
245
246
  end
246
247
  end
247
- false
248
+ @configuration.soft_validations
248
249
  end
249
250
  end
250
251
 
@@ -50,12 +50,24 @@ module NcsNavigator::Warehouse::Transformers
50
50
  private
51
51
 
52
52
  # TODO: share this stuff between here and EnumTransformer
53
-
54
53
  def save(record, status)
55
- if record.valid?
54
+ if !(valid = record.valid?)
55
+ log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
56
+ record.errors.keys.each do |prop|
57
+ record.errors[prop].each do |e|
58
+ status.unsuccessful_record(
59
+ record, "Invalid after transformation: #{e}.",
60
+ :attribute_name => prop,
61
+ :attribute_value => record.send(prop).inspect
62
+ )
63
+ end
64
+ end
65
+ end
66
+
67
+ if valid || @configuration.soft_validations
56
68
  log.debug("Saving valid transformed event record #{record_ident record}.")
57
69
  begin
58
- unless record.save
70
+ unless (@configuration.soft_validations ? record.save! : record.save)
59
71
  msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
60
72
  log.error msg
61
73
  status.unsuccessful_record(record, msg)
@@ -65,17 +77,6 @@ module NcsNavigator::Warehouse::Transformers
65
77
  log.error msg
66
78
  status.unsuccessful_record(record, msg)
67
79
  end
68
- else
69
- log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
70
- record.errors.keys.each do |prop|
71
- record.errors[prop].each do |e|
72
- status.unsuccessful_record(
73
- record, "Invalid after transformation: #{e}.",
74
- :attribute_name => prop,
75
- :attribute_value => record.send(prop).inspect
76
- )
77
- end
78
- end
79
80
  end
80
81
  end
81
82
 
@@ -95,4 +96,3 @@ module NcsNavigator::Warehouse::Transformers
95
96
  end
96
97
  end
97
98
  end
98
-
@@ -68,6 +68,7 @@ module NcsNavigator::Warehouse::Transformers
68
68
  def initialize(config, exec_and_args, options={})
69
69
  @configuration = config
70
70
  @exec_and_args = exec_and_args
71
+ @exec_and_args << '--soft_validations' if @configuration.soft_validations
71
72
  @directory = options[:directory] || '.'
72
73
  @environment = options[:environment] || {}
73
74
  end
@@ -1,5 +1,5 @@
1
1
  module NcsNavigator
2
2
  module Warehouse
3
- VERSION = '0.14.0'
3
+ VERSION = '0.15.0.pre1'
4
4
  end
5
5
  end
@@ -48,5 +48,5 @@ Gem::Specification.new do |s|
48
48
  s.add_development_dependency 'rake', '~> 0.9.2'
49
49
  s.add_development_dependency 'yard', '~> 0.7.2'
50
50
  s.add_development_dependency 'ci_reporter', '1.6.6'
51
- s.add_development_dependency 'fakefs', '0.4.0' # FakeFS does not follow semver
51
+ s.add_development_dependency 'fakefs', '0.4.2' # FakeFS does not follow semver
52
52
  end
@@ -2,6 +2,33 @@ require 'spec_helper'
2
2
 
3
3
  module NcsNavigator::Warehouse
4
4
  describe DatabaseInitializer do
5
+ describe '#drop_all_null_constraints', :slow do
6
+ it 'drops NOT NULL constraints on all non-primary key columns in all tables' do
7
+ subject = DatabaseInitializer.new(spec_config)
8
+ subject.set_up_repository(:working)
9
+ subject.replace_schema
10
+ subject.drop_all_null_constraints(:working)
11
+
12
+ models = ::DataMapper::Model.descendants
13
+ adapter = ::DataMapper.repository(:mdes_warehouse_working).adapter
14
+
15
+ models.each do |m|
16
+ nullables = adapter.select(%Q{
17
+ SELECT cs.column_name
18
+ FROM information_schema.columns AS cs
19
+ WHERE cs.is_nullable = 'YES'
20
+ AND cs.table_name = '#{m.storage_name}'
21
+ })
22
+
23
+ m.properties.each do |p|
24
+ unless m.key.include?(p)
25
+ nullables.should include(p.name.to_s)
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+
5
32
  describe '#replace_schema', :slow, :use_mdes do
6
33
  subject { DatabaseInitializer.new(spec_config) }
7
34
  let(:mdes_models) { spec_config.models_module.mdes_order }
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ncs_mdes_warehouse
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.0
5
- prerelease:
4
+ version: 0.15.0.pre1
5
+ prerelease: 7
6
6
  platform: ruby
7
7
  authors:
8
8
  - Rhett Sutphin
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-05-15 00:00:00.000000000 Z
12
+ date: 2013-06-22 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: ncs_mdes
@@ -452,7 +452,7 @@ dependencies:
452
452
  requirements:
453
453
  - - '='
454
454
  - !ruby/object:Gem::Version
455
- version: 0.4.0
455
+ version: 0.4.2
456
456
  type: :development
457
457
  prerelease: false
458
458
  version_requirements: !ruby/object:Gem::Requirement
@@ -460,7 +460,7 @@ dependencies:
460
460
  requirements:
461
461
  - - '='
462
462
  - !ruby/object:Gem::Version
463
- version: 0.4.0
463
+ version: 0.4.2
464
464
  description:
465
465
  email:
466
466
  - r-sutphin@northwestern.edu
@@ -3669,16 +3669,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
3669
3669
  version: '0'
3670
3670
  segments:
3671
3671
  - 0
3672
- hash: -3845842505256535190
3672
+ hash: -4565215284799470018
3673
3673
  required_rubygems_version: !ruby/object:Gem::Requirement
3674
3674
  none: false
3675
3675
  requirements:
3676
- - - ! '>='
3676
+ - - ! '>'
3677
3677
  - !ruby/object:Gem::Version
3678
- version: '0'
3679
- segments:
3680
- - 0
3681
- hash: -3845842505256535190
3678
+ version: 1.3.1
3682
3679
  requirements: []
3683
3680
  rubyforge_project:
3684
3681
  rubygems_version: 1.8.25