ncs_mdes_warehouse 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +31 -2
- data/README.md +25 -0
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/drf_therm_verification.rb +1 -1
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment.rb +4 -4
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment_prob_log.rb +3 -3
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/prec_therm_cert.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/ref_freezer_verification.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/sample_receipt_confirm.rb +1 -1
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/spec_equipment.rb +4 -4
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/spec_spsc_info.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/srsc_info.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/trh_meter_calibration.rb +1 -1
- data/lib/ncs_navigator/warehouse/database_initializer.rb +112 -31
- data/lib/ncs_navigator/warehouse/table_modeler/mdes_ext.rb +1 -1
- data/lib/ncs_navigator/warehouse/transform_load.rb +15 -9
- data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb +26 -6
- data/lib/ncs_navigator/warehouse/version.rb +1 -1
- data/lib/ncs_navigator/warehouse/xml_emitter.rb +8 -7
- data/spec/ncs_navigator/warehouse/database_initializer_spec.rb +113 -0
- data/spec/ncs_navigator/warehouse/table_modeler_spec.rb +7 -0
- data/spec/ncs_navigator/warehouse/transform_load_spec.rb +28 -0
- data/spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml +12 -4
- data/spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb +18 -1
- data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb +8 -2
- data/spec/spec_helper.rb +2 -1
- metadata +53 -51
data/CHANGELOG.md
CHANGED
|
@@ -1,13 +1,42 @@
|
|
|
1
1
|
NCS Navigator MDES Warehouse History
|
|
2
2
|
====================================
|
|
3
3
|
|
|
4
|
+
0.4.0
|
|
5
|
+
-----
|
|
6
|
+
|
|
7
|
+
- Restore creation of foreign key constraints during schema
|
|
8
|
+
initialization. This was lost after upgrading to DataMapper 1.2 in
|
|
9
|
+
the prehistory of the project. (#1639)
|
|
10
|
+
|
|
11
|
+
- Skip more unknown/missing codes when reading VDR XML. The reader now
|
|
12
|
+
treats any non-coded, non-required field whose value is exactly -3,
|
|
13
|
+
-4, -6, or -7 as NULL. (#1702)
|
|
14
|
+
|
|
15
|
+
- Skip entire records where the PK is one of the unknown/missing
|
|
16
|
+
codes. (#1703)
|
|
17
|
+
|
|
18
|
+
- Prevent data integrity errors from killing the entire ETL
|
|
19
|
+
process. They are now recorded as transform errors just like
|
|
20
|
+
validation problems. (#1717)
|
|
21
|
+
|
|
22
|
+
- Ignore blank PII values when generating the `:pii` meta attribute
|
|
23
|
+
for model properties. (Done with #1814)
|
|
24
|
+
|
|
25
|
+
- Create `no_pii` schema containing views with all PII excised. If
|
|
26
|
+
there is a role named `mdes_warehouse_no_pii`, it will automatically
|
|
27
|
+
be granted SELECT on the content of this schema. (#1814)
|
|
28
|
+
|
|
29
|
+
- Changed default emit-xml filename when PII is included. (#1816)
|
|
30
|
+
|
|
4
31
|
0.3.2
|
|
5
32
|
-----
|
|
6
33
|
|
|
7
|
-
- Correct VDR XML reader for the case
|
|
8
|
-
|
|
34
|
+
- Correct VDR XML reader for the case of an MDES table that contains a
|
|
35
|
+
variable with the same name as the table. (#1695)
|
|
36
|
+
|
|
9
37
|
- Include the Study Center record in the instances created from the
|
|
10
38
|
`SamplingUnits` transformer. (#1690)
|
|
39
|
+
|
|
11
40
|
- Change the default compare level to 3. It turns out that the full
|
|
12
41
|
compare is not that slow, so there's no reason to default it to 1.
|
|
13
42
|
(#1691)
|
data/README.md
CHANGED
|
@@ -63,6 +63,31 @@ ways to create them. `createdb` is one:
|
|
|
63
63
|
$ createdb -e -O mdes_warehouse mdes_warehouse_reporting
|
|
64
64
|
CREATE DATABASE mdes_warehouse_reporting OWNER mdes_warehouse;
|
|
65
65
|
|
|
66
|
+
#### Additional database roles
|
|
67
|
+
|
|
68
|
+
For safe reporting purposes, the warehouse automatically creates
|
|
69
|
+
PII-screened views in a schema named `no_pii`. Since those views are
|
|
70
|
+
dropped along with everything else when the warehouse schema is
|
|
71
|
+
recreated, direct grants from user accounts to to the views will be
|
|
72
|
+
lost every time ETL runs. To work around this potential problem, the
|
|
73
|
+
schema creation process looks for a role named
|
|
74
|
+
`mdes_warehouse_no_pii`. If this role exists, it will be granted
|
|
75
|
+
read-only access to these PII-screened views. Here's how you can
|
|
76
|
+
create the role:
|
|
77
|
+
|
|
78
|
+
$ createuser -e -h dbserver.my.org -LRSD mdes_warehouse_no_pii
|
|
79
|
+
CREATE ROLE mdes_warehouse_no_pii NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN;
|
|
80
|
+
|
|
81
|
+
Individual data analyst accounts can then be granted this role to have
|
|
82
|
+
access to a PII-free view of the warehouse:
|
|
83
|
+
|
|
84
|
+
# In a database console
|
|
85
|
+
GRANT mdes_warehouse_no_pii TO some_data_analyst;
|
|
86
|
+
# Following is optional; it will make sure that some_data_analyst
|
|
87
|
+
# will default to querying the PII-free views instead of the
|
|
88
|
+
# (inaccessible) full-data tables in the public schema.
|
|
89
|
+
ALTER some_data_analyst SET search_path=no_pii;
|
|
90
|
+
|
|
66
91
|
### Configure bcdatabase
|
|
67
92
|
|
|
68
93
|
Like the rest of the NCS Navigator suite, MDES Warehouse uses
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/drf_therm_verification.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -12,22 +12,22 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
19
19
|
property :equip_id,
|
|
20
20
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
21
|
-
{ :key => true, :required => true, :
|
|
21
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
22
22
|
property :equipment_type,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :required => true, :
|
|
24
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-4"] }
|
|
25
25
|
property :equipment_type_oth,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
27
|
{ :pii => :possible, :length => 0..255 }
|
|
28
28
|
property :serial_no,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
|
-
{ :required => true, :
|
|
30
|
+
{ :required => true, :length => 1..50 }
|
|
31
31
|
property :government_asset_tag_no,
|
|
32
32
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
33
33
|
{ :length => 0..36 }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment_prob_log.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -21,10 +21,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
21
21
|
:child_key => [ :equip_id ], :required => true
|
|
22
22
|
property :problem_id,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :key => true, :required => true, :
|
|
24
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
25
25
|
property :equipment_type,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :required => true, :
|
|
27
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-4"] }
|
|
28
28
|
property :equipment_type_oth,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :pii => :possible, :length => 0..255 }
|
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -24,7 +24,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
24
24
|
:child_key => [ :staff_id ], :required => true
|
|
25
25
|
property :certification_id,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :key => true, :required => true, :
|
|
27
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
28
28
|
property :precision_cert_status,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :required => true, :length => 1..2, :set => ["1", "2", "-4"] }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/ref_freezer_verification.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -24,7 +24,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
24
24
|
:child_key => [ :staff_id ], :required => true
|
|
25
25
|
property :verification_id,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :key => true, :required => true, :
|
|
27
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
28
28
|
property :verification_dt,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :required => true, :length => 1..10, :format => /^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])?$/ }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/sample_receipt_confirm.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -12,22 +12,22 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :spsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SpecSpscInfo',
|
|
18
18
|
:child_key => [ :spsc_id ], :required => true
|
|
19
19
|
property :equip_id,
|
|
20
20
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
21
|
-
{ :key => true, :required => true, :
|
|
21
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
22
22
|
property :equipment_type,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :required => true, :
|
|
24
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-7", "-4"] }
|
|
25
25
|
property :equipment_type_oth,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
27
|
{ :pii => :possible, :length => 0..255 }
|
|
28
28
|
property :serial_no,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
|
-
{ :
|
|
30
|
+
{ :length => 0..50 }
|
|
31
31
|
property :government_asset_tag_no,
|
|
32
32
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
33
33
|
{ :length => 0..36 }
|
|
@@ -12,10 +12,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
property :spsc_id,
|
|
17
17
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
18
|
-
{ :key => true, :required => true, :
|
|
18
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
19
19
|
belongs_to :address,
|
|
20
20
|
'NcsNavigator::Warehouse::Models::TwoPointZero::Address',
|
|
21
21
|
:child_key => [ :address_id ], :required => false
|
|
@@ -12,10 +12,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
property :srsc_id,
|
|
17
17
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
18
|
-
{ :key => true, :required => true, :
|
|
18
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
19
19
|
belongs_to :address,
|
|
20
20
|
'NcsNavigator::Warehouse::Models::TwoPointZero::Address',
|
|
21
21
|
:child_key => [ :address_id ], :required => false
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/trh_meter_calibration.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -41,13 +41,13 @@ module NcsNavigator::Warehouse
|
|
|
41
41
|
# @param mode [:reporting, :working, :both] which repositories to
|
|
42
42
|
# set up. In general, the working database should only be needed
|
|
43
43
|
# internal to the warehouse's scripts. The `:default` repo will
|
|
44
|
-
# be the
|
|
44
|
+
# be the working repo unless only the reporting repo is
|
|
45
45
|
# configured in.
|
|
46
46
|
# @return [void]
|
|
47
47
|
def set_up_repository(mode=:reporting, prefix="mdes_warehouse")
|
|
48
48
|
fail "Invalid mode #{mode.inspect}" unless [:reporting, :working, :both].include?(mode)
|
|
49
49
|
modes = case mode
|
|
50
|
-
when :both then [:
|
|
50
|
+
when :both then [:working, :reporting]
|
|
51
51
|
else [mode]
|
|
52
52
|
end
|
|
53
53
|
connect_one(modes.first, :default)
|
|
@@ -83,24 +83,57 @@ module NcsNavigator::Warehouse
|
|
|
83
83
|
|
|
84
84
|
shell.say "Loading MDES models..."
|
|
85
85
|
log.info "Initializing schema for MDES #{configuration.mdes.specification_version}"
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
shell.clear_line_then_say "Adding #{m.storage_name(:mdes_warehouse_working)}..."
|
|
90
|
-
m.auto_migrate!(:mdes_warehouse_working)
|
|
91
|
-
end
|
|
86
|
+
shell.clear_line_then_say("Creating schema in working database...")
|
|
87
|
+
::DataMapper.auto_migrate!
|
|
88
|
+
|
|
92
89
|
shell.clear_line_then_say(
|
|
93
90
|
"Added #{configuration.models_module.mdes_order.size} MDES tables.\n")
|
|
91
|
+
|
|
92
|
+
create_no_pii_views
|
|
94
93
|
end
|
|
95
94
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
# @private Exposed for use in tests
|
|
96
|
+
def drop_all(which, options={})
|
|
97
|
+
shell.clear_line_then_say "Dropping everything in #{which} schema" unless options[:quiet]
|
|
98
|
+
log.info "Dropping everything in #{which} schema..."
|
|
99
99
|
::DataMapper.repository(:"mdes_warehouse_#{which}").adapter.
|
|
100
100
|
execute("DROP OWNED BY #{params(which)['username']}")
|
|
101
|
-
shell.clear_line_then_say "Dropped everything in #{which} schema.\n"
|
|
101
|
+
shell.clear_line_then_say "Dropped everything in #{which} schema.\n" unless options[:quiet]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def create_no_pii_views
|
|
105
|
+
no_pii_schema = 'no_pii'
|
|
106
|
+
shell.say("Creating no-PII views...")
|
|
107
|
+
|
|
108
|
+
views = configuration.models_module.mdes_order.collect do |model|
|
|
109
|
+
selects = model.properties.collect { |p|
|
|
110
|
+
[p.pii ? "varchar(32) '[pii=#{p.pii}]'" : p.name, p.name].join(' AS ')
|
|
111
|
+
}
|
|
112
|
+
%Q(CREATE VIEW #{no_pii_schema}.#{model.mdes_table_name}
|
|
113
|
+
AS SELECT #{selects.join(',')} FROM public.#{model.mdes_table_name})
|
|
114
|
+
end
|
|
115
|
+
::DataMapper.repository(:mdes_warehouse_working).adapter.
|
|
116
|
+
execute("CREATE SCHEMA #{no_pii_schema} #{views.join("\n")}")
|
|
117
|
+
shell.clear_line_then_say("Created no-PII views.\n")
|
|
118
|
+
|
|
119
|
+
role_name = 'mdes_warehouse_no_pii'
|
|
120
|
+
role_exists = ::DataMapper.repository(:mdes_warehouse_working).adapter.
|
|
121
|
+
select("SELECT rolname FROM pg_catalog.pg_roles WHERE rolname='#{role_name}'").size > 0
|
|
122
|
+
if role_exists
|
|
123
|
+
shell.say("- Granting SELECT on no-PII views to #{role_name}...")
|
|
124
|
+
[
|
|
125
|
+
"GRANT USAGE ON SCHEMA #{no_pii_schema} TO #{role_name}",
|
|
126
|
+
"GRANT SELECT ON ALL TABLES IN SCHEMA #{no_pii_schema} TO #{role_name}"
|
|
127
|
+
].each do |stmt|
|
|
128
|
+
::DataMapper.repository(:mdes_warehouse_working).adapter.execute(stmt)
|
|
129
|
+
end
|
|
130
|
+
shell.clear_line_then_say("- Granted SELECT on no-PII views to #{role_name}.\n")
|
|
131
|
+
else
|
|
132
|
+
shell.say_line(
|
|
133
|
+
"- Database role #{role_name} does not exist. Performing no grants for no-PII views.")
|
|
134
|
+
end
|
|
102
135
|
end
|
|
103
|
-
private :
|
|
136
|
+
private :create_no_pii_views
|
|
104
137
|
|
|
105
138
|
##
|
|
106
139
|
# Replaces the reporting database with a clone of the working
|
|
@@ -117,36 +150,84 @@ module NcsNavigator::Warehouse
|
|
|
117
150
|
pgpass.update params(:reporting)
|
|
118
151
|
end
|
|
119
152
|
|
|
120
|
-
|
|
153
|
+
shell.say 'Cloning working schema into reporting schema...'
|
|
154
|
+
|
|
155
|
+
dump_file = Tempfile.new('wh_clone_dump')
|
|
156
|
+
|
|
157
|
+
dump_cmd = escape_cmd([
|
|
121
158
|
configuration.pg_bin('pg_dump'),
|
|
122
159
|
pg_params(params(:working)),
|
|
123
160
|
'--format=custom',
|
|
161
|
+
"--file=#{dump_file.path}",
|
|
124
162
|
params(:working)['database']
|
|
125
|
-
].flatten
|
|
163
|
+
].flatten)
|
|
164
|
+
|
|
165
|
+
shell.clear_line_then_say(
|
|
166
|
+
"Cloning working to reporting... dumping working database to temporary file")
|
|
167
|
+
log.debug("Dump command: #{dump_cmd}")
|
|
168
|
+
unless system(dump_cmd)
|
|
169
|
+
shell.say(
|
|
170
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
171
|
+
log.error('Dump failed.')
|
|
172
|
+
return false
|
|
173
|
+
end
|
|
126
174
|
|
|
127
|
-
|
|
175
|
+
list_file = Tempfile.new('wh_clone_list')
|
|
176
|
+
list_cmd = escape_cmd([
|
|
177
|
+
configuration.pg_bin('pg_restore'),
|
|
178
|
+
pg_params(params(:working)),
|
|
179
|
+
'--list',
|
|
180
|
+
"--file=#{list_file.path}",
|
|
181
|
+
dump_file.path
|
|
182
|
+
].flatten)
|
|
183
|
+
|
|
184
|
+
shell.clear_line_then_say(
|
|
185
|
+
"Cloning working to reporting... extracting dump contents")
|
|
186
|
+
log.debug("List command: #{list_cmd}")
|
|
187
|
+
unless system(list_cmd)
|
|
188
|
+
shell.say(
|
|
189
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
190
|
+
log.error('Dump failed.')
|
|
191
|
+
return false
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
shell.clear_line_then_say(
|
|
195
|
+
"Cloning working to reporting... filtering content list")
|
|
196
|
+
list_file.rewind
|
|
197
|
+
selection_file = Tempfile.new('wh_clone_selection')
|
|
198
|
+
list_file.read.split("\n").each do |line|
|
|
199
|
+
if line =~ %r{#{params(:working)['username']}$}
|
|
200
|
+
selection_file.puts line
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
restore_cmd = escape_cmd([
|
|
128
205
|
configuration.pg_bin('pg_restore'),
|
|
129
206
|
pg_params(params(:reporting)),
|
|
130
|
-
|
|
131
|
-
'--dbname', params(:reporting)['database']
|
|
132
|
-
|
|
207
|
+
"--use-list=#{selection_file.path}",
|
|
208
|
+
'--dbname', params(:reporting)['database'],
|
|
209
|
+
dump_file.path
|
|
210
|
+
].flatten)
|
|
133
211
|
|
|
134
|
-
drop_all(:reporting)
|
|
212
|
+
drop_all(:reporting, :quiet => true)
|
|
135
213
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
log.
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
shell.clear_line_then_say(
|
|
142
|
-
"Clone from working to reporting failed. See above for detail.\n")
|
|
143
|
-
log.error('Clone failed.')
|
|
144
|
-
return false
|
|
145
|
-
else
|
|
146
|
-
shell.clear_line_then_say("Clone from working to reporting successful.\n")
|
|
214
|
+
shell.clear_line_then_say(
|
|
215
|
+
"Cloning working to reporting... loading filtered dump into reporting database")
|
|
216
|
+
log.debug("Restore command: #{restore_cmd.inspect}")
|
|
217
|
+
if system(restore_cmd)
|
|
218
|
+
shell.clear_line_then_say("Successfully cloned working to reporting.\n")
|
|
147
219
|
log.info('Clone succeeded.')
|
|
148
220
|
return true
|
|
221
|
+
else
|
|
222
|
+
shell.say(
|
|
223
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
224
|
+
log.error('Dump failed.')
|
|
225
|
+
return false
|
|
149
226
|
end
|
|
227
|
+
ensure
|
|
228
|
+
dump_file.unlink if dump_file
|
|
229
|
+
list_file.unlink if list_file
|
|
230
|
+
selection_file.unlink if selection_file
|
|
150
231
|
end
|
|
151
232
|
|
|
152
233
|
def pg_params(p)
|
|
@@ -25,16 +25,22 @@ module NcsNavigator::Warehouse
|
|
|
25
25
|
def repo.identity_map(model); {}; end
|
|
26
26
|
|
|
27
27
|
build_status_for(transformer, position).tap do |status|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
repo.adapter.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
28
|
+
begin
|
|
29
|
+
TransformStatus.transaction do
|
|
30
|
+
if repo.adapter.to_s =~ /Postgres/
|
|
31
|
+
repo.adapter.execute("SET LOCAL synchronous_commit TO OFF")
|
|
32
|
+
end
|
|
33
|
+
begin
|
|
34
|
+
transformer.transform(status)
|
|
35
|
+
rescue => e
|
|
36
|
+
shell.say_line("\nTransform failed. (See log for more detail.)")
|
|
37
|
+
status.add_error("Transform failed. #{e.class}: #{e}.")
|
|
38
|
+
end
|
|
37
39
|
end
|
|
40
|
+
rescue DataObjects::IntegrityError => e
|
|
41
|
+
shell.say_line("\nTransform failed with data integrity error. (See log for more detail.)")
|
|
42
|
+
log.error("Transform failed with data integrity error: #{e}.")
|
|
43
|
+
status.add_error("Transform failed with data integrity error: #{e}.")
|
|
38
44
|
end
|
|
39
45
|
status.end_time = Time.now
|
|
40
46
|
unless status.save
|