ncs_mdes_warehouse 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +31 -2
- data/README.md +25 -0
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/drf_therm_verification.rb +1 -1
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment.rb +4 -4
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment_prob_log.rb +3 -3
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/prec_therm_cert.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/ref_freezer_verification.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/sample_receipt_confirm.rb +1 -1
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/spec_equipment.rb +4 -4
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/spec_spsc_info.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/srsc_info.rb +2 -2
- data/generated_models/ncs_navigator/warehouse/models/two_point_zero/trh_meter_calibration.rb +1 -1
- data/lib/ncs_navigator/warehouse/database_initializer.rb +112 -31
- data/lib/ncs_navigator/warehouse/table_modeler/mdes_ext.rb +1 -1
- data/lib/ncs_navigator/warehouse/transform_load.rb +15 -9
- data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb +26 -6
- data/lib/ncs_navigator/warehouse/version.rb +1 -1
- data/lib/ncs_navigator/warehouse/xml_emitter.rb +8 -7
- data/spec/ncs_navigator/warehouse/database_initializer_spec.rb +113 -0
- data/spec/ncs_navigator/warehouse/table_modeler_spec.rb +7 -0
- data/spec/ncs_navigator/warehouse/transform_load_spec.rb +28 -0
- data/spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml +12 -4
- data/spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb +18 -1
- data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb +8 -2
- data/spec/spec_helper.rb +2 -1
- metadata +53 -51
data/CHANGELOG.md
CHANGED
|
@@ -1,13 +1,42 @@
|
|
|
1
1
|
NCS Navigator MDES Warehouse History
|
|
2
2
|
====================================
|
|
3
3
|
|
|
4
|
+
0.4.0
|
|
5
|
+
-----
|
|
6
|
+
|
|
7
|
+
- Restore creation of foreign key constraints during schema
|
|
8
|
+
initialization. This was lost after upgrading to DataMapper 1.2 in
|
|
9
|
+
the prehistory of the project. (#1639)
|
|
10
|
+
|
|
11
|
+
- Skip more unknown/missing codes when reading VDR XML. The reader now
|
|
12
|
+
treats any non-coded, non-required field whose value is exactly -3,
|
|
13
|
+
-4, -6, or -7 as NULL. (#1702)
|
|
14
|
+
|
|
15
|
+
- Skip entire records where the PK is one of the unknown/missing
|
|
16
|
+
codes. (#1703)
|
|
17
|
+
|
|
18
|
+
- Prevent data integrity errors from killing the entire ETL
|
|
19
|
+
process. They are now recorded as transform errors just like
|
|
20
|
+
validation problems. (#1717)
|
|
21
|
+
|
|
22
|
+
- Ignore blank PII values when generating the `:pii` meta attribute
|
|
23
|
+
for model properties. (Done with #1814)
|
|
24
|
+
|
|
25
|
+
- Create `no_pii` schema containing views with all PII excised. If
|
|
26
|
+
there is a role named `mdes_warehouse_no_pii`, it will automatically
|
|
27
|
+
be granted SELECT on the content of this schema. (#1814)
|
|
28
|
+
|
|
29
|
+
- Changed default emit-xml filename when PII is included. (#1816)
|
|
30
|
+
|
|
4
31
|
0.3.2
|
|
5
32
|
-----
|
|
6
33
|
|
|
7
|
-
- Correct VDR XML reader for the case
|
|
8
|
-
|
|
34
|
+
- Correct VDR XML reader for the case of an MDES table that contains a
|
|
35
|
+
variable with the same name as the table. (#1695)
|
|
36
|
+
|
|
9
37
|
- Include the Study Center record in the instances created from the
|
|
10
38
|
`SamplingUnits` transformer. (#1690)
|
|
39
|
+
|
|
11
40
|
- Change the default compare level to 3. It turns out that the full
|
|
12
41
|
compare is not that slow, so there's no reason to default it to 1.
|
|
13
42
|
(#1691)
|
data/README.md
CHANGED
|
@@ -63,6 +63,31 @@ ways to create them. `createdb` is one:
|
|
|
63
63
|
$ createdb -e -O mdes_warehouse mdes_warehouse_reporting
|
|
64
64
|
CREATE DATABASE mdes_warehouse_reporting OWNER mdes_warehouse;
|
|
65
65
|
|
|
66
|
+
#### Additional database roles
|
|
67
|
+
|
|
68
|
+
For safe reporting purposes, the warehouse automatically creates
|
|
69
|
+
PII-screened views in a schema named `no_pii`. Since those views are
|
|
70
|
+
dropped along with everything else when the warehouse schema is
|
|
71
|
+
recreated, direct grants from user accounts to to the views will be
|
|
72
|
+
lost every time ETL runs. To work around this potential problem, the
|
|
73
|
+
schema creation process looks for a role named
|
|
74
|
+
`mdes_warehouse_no_pii`. If this role exists, it will be granted
|
|
75
|
+
read-only access to these PII-screened views. Here's how you can
|
|
76
|
+
create the role:
|
|
77
|
+
|
|
78
|
+
$ createuser -e -h dbserver.my.org -LRSD mdes_warehouse_no_pii
|
|
79
|
+
CREATE ROLE mdes_warehouse_no_pii NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN;
|
|
80
|
+
|
|
81
|
+
Individual data analyst accounts can then be granted this role to have
|
|
82
|
+
access to a PII-free view of the warehouse:
|
|
83
|
+
|
|
84
|
+
# In a database console
|
|
85
|
+
GRANT mdes_warehouse_no_pii TO some_data_analyst;
|
|
86
|
+
# Following is optional; it will make sure that some_data_analyst
|
|
87
|
+
# will default to querying the PII-free views instead of the
|
|
88
|
+
# (inaccessible) full-data tables in the public schema.
|
|
89
|
+
ALTER some_data_analyst SET search_path=no_pii;
|
|
90
|
+
|
|
66
91
|
### Configure bcdatabase
|
|
67
92
|
|
|
68
93
|
Like the rest of the NCS Navigator suite, MDES Warehouse uses
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/drf_therm_verification.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -12,22 +12,22 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
19
19
|
property :equip_id,
|
|
20
20
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
21
|
-
{ :key => true, :required => true, :
|
|
21
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
22
22
|
property :equipment_type,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :required => true, :
|
|
24
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-4"] }
|
|
25
25
|
property :equipment_type_oth,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
27
|
{ :pii => :possible, :length => 0..255 }
|
|
28
28
|
property :serial_no,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
|
-
{ :required => true, :
|
|
30
|
+
{ :required => true, :length => 1..50 }
|
|
31
31
|
property :government_asset_tag_no,
|
|
32
32
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
33
33
|
{ :length => 0..36 }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment_prob_log.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -21,10 +21,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
21
21
|
:child_key => [ :equip_id ], :required => true
|
|
22
22
|
property :problem_id,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :key => true, :required => true, :
|
|
24
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
25
25
|
property :equipment_type,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :required => true, :
|
|
27
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-4"] }
|
|
28
28
|
property :equipment_type_oth,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :pii => :possible, :length => 0..255 }
|
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -24,7 +24,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
24
24
|
:child_key => [ :staff_id ], :required => true
|
|
25
25
|
property :certification_id,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :key => true, :required => true, :
|
|
27
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
28
28
|
property :precision_cert_status,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :required => true, :length => 1..2, :set => ["1", "2", "-4"] }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/ref_freezer_verification.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -24,7 +24,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
24
24
|
:child_key => [ :staff_id ], :required => true
|
|
25
25
|
property :verification_id,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
|
-
{ :key => true, :required => true, :
|
|
27
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
28
28
|
property :verification_dt,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
30
|
{ :required => true, :length => 1..10, :format => /^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])?$/ }
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/sample_receipt_confirm.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -12,22 +12,22 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :spsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SpecSpscInfo',
|
|
18
18
|
:child_key => [ :spsc_id ], :required => true
|
|
19
19
|
property :equip_id,
|
|
20
20
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
21
|
-
{ :key => true, :required => true, :
|
|
21
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
22
22
|
property :equipment_type,
|
|
23
23
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
24
|
-
{ :required => true, :
|
|
24
|
+
{ :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-7", "-4"] }
|
|
25
25
|
property :equipment_type_oth,
|
|
26
26
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
27
27
|
{ :pii => :possible, :length => 0..255 }
|
|
28
28
|
property :serial_no,
|
|
29
29
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
30
|
-
{ :
|
|
30
|
+
{ :length => 0..50 }
|
|
31
31
|
property :government_asset_tag_no,
|
|
32
32
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
33
33
|
{ :length => 0..36 }
|
|
@@ -12,10 +12,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
property :spsc_id,
|
|
17
17
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
18
|
-
{ :key => true, :required => true, :
|
|
18
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
19
19
|
belongs_to :address,
|
|
20
20
|
'NcsNavigator::Warehouse::Models::TwoPointZero::Address',
|
|
21
21
|
:child_key => [ :address_id ], :required => false
|
|
@@ -12,10 +12,10 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
property :srsc_id,
|
|
17
17
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
18
|
-
{ :key => true, :required => true, :
|
|
18
|
+
{ :key => true, :required => true, :length => 1..36 }
|
|
19
19
|
belongs_to :address,
|
|
20
20
|
'NcsNavigator::Warehouse::Models::TwoPointZero::Address',
|
|
21
21
|
:child_key => [ :address_id ], :required => false
|
data/generated_models/ncs_navigator/warehouse/models/two_point_zero/trh_meter_calibration.rb
CHANGED
|
@@ -12,7 +12,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
|
|
|
12
12
|
|
|
13
13
|
property :psu_id,
|
|
14
14
|
NcsNavigator::Warehouse::DataMapper::NcsString,
|
|
15
|
-
{ :required => true, :
|
|
15
|
+
{ :required => true, :length => 2..8, :set => ["20000054", "20000032", "20000032", "20000032", "20000032", "20000016", "20000039", "20000200", "20000028", "20000063", "20000067", "20000201", "20000202", "20000203", "20000090", "20000083", "20000204", "20000205", "20000042", "20000037", "20000206", "20000040", "20000207", "20000208", "20000209", "20000091", "20000210", "20000069", "20000211", "20000094", "20000212", "20000213", "20000102", "20000214", "20000215", "20000044", "20000216", "20000216", "20000216", "20000030", "20000217", "20000218", "20000218", "20000218", "20000219", "20000220", "20000221", "20000092", "20000222", "20000223", "20000224", "20000225", "20000225", "20000088", "20000087", "20000226", "20000103", "20000227", "20000228", "20000229", "20000230", "20000231", "20000232", "20000025", "20000233", "20000233", "20000233", "20000048", "20000234", "20000235", "20000050", "20000236", "20000035", "20000237", "20000238", "20000239", "20000240", "20000052", "20000241", "20000243", "20000244", "20000245", "20000246", "20000247", "20000248", "20000113", "20000249", "20000250", "20000251", "20000086", "20000252", "20000253", "20000254", "20000255", "20000256", "20000018", "20000058", "20000014", "20000257", "20000258", "20000259", "20000259", "20000098", "20000060", "20000260", "20000260", "20000260", "20000260", "20000261", "20000262", "20000263", "20000097", "20000264", "20000264", "20000265", "20000062", "20000117", "20000266", "20000267", "20000268", "20000269", "20000270", "20000271", "20000272", "20000273", "-4", "20000000"] }
|
|
16
16
|
belongs_to :srsc,
|
|
17
17
|
'NcsNavigator::Warehouse::Models::TwoPointZero::SrscInfo',
|
|
18
18
|
:child_key => [ :srsc_id ], :required => true
|
|
@@ -41,13 +41,13 @@ module NcsNavigator::Warehouse
|
|
|
41
41
|
# @param mode [:reporting, :working, :both] which repositories to
|
|
42
42
|
# set up. In general, the working database should only be needed
|
|
43
43
|
# internal to the warehouse's scripts. The `:default` repo will
|
|
44
|
-
# be the
|
|
44
|
+
# be the working repo unless only the reporting repo is
|
|
45
45
|
# configured in.
|
|
46
46
|
# @return [void]
|
|
47
47
|
def set_up_repository(mode=:reporting, prefix="mdes_warehouse")
|
|
48
48
|
fail "Invalid mode #{mode.inspect}" unless [:reporting, :working, :both].include?(mode)
|
|
49
49
|
modes = case mode
|
|
50
|
-
when :both then [:
|
|
50
|
+
when :both then [:working, :reporting]
|
|
51
51
|
else [mode]
|
|
52
52
|
end
|
|
53
53
|
connect_one(modes.first, :default)
|
|
@@ -83,24 +83,57 @@ module NcsNavigator::Warehouse
|
|
|
83
83
|
|
|
84
84
|
shell.say "Loading MDES models..."
|
|
85
85
|
log.info "Initializing schema for MDES #{configuration.mdes.specification_version}"
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
shell.clear_line_then_say "Adding #{m.storage_name(:mdes_warehouse_working)}..."
|
|
90
|
-
m.auto_migrate!(:mdes_warehouse_working)
|
|
91
|
-
end
|
|
86
|
+
shell.clear_line_then_say("Creating schema in working database...")
|
|
87
|
+
::DataMapper.auto_migrate!
|
|
88
|
+
|
|
92
89
|
shell.clear_line_then_say(
|
|
93
90
|
"Added #{configuration.models_module.mdes_order.size} MDES tables.\n")
|
|
91
|
+
|
|
92
|
+
create_no_pii_views
|
|
94
93
|
end
|
|
95
94
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
# @private Exposed for use in tests
|
|
96
|
+
def drop_all(which, options={})
|
|
97
|
+
shell.clear_line_then_say "Dropping everything in #{which} schema" unless options[:quiet]
|
|
98
|
+
log.info "Dropping everything in #{which} schema..."
|
|
99
99
|
::DataMapper.repository(:"mdes_warehouse_#{which}").adapter.
|
|
100
100
|
execute("DROP OWNED BY #{params(which)['username']}")
|
|
101
|
-
shell.clear_line_then_say "Dropped everything in #{which} schema.\n"
|
|
101
|
+
shell.clear_line_then_say "Dropped everything in #{which} schema.\n" unless options[:quiet]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def create_no_pii_views
|
|
105
|
+
no_pii_schema = 'no_pii'
|
|
106
|
+
shell.say("Creating no-PII views...")
|
|
107
|
+
|
|
108
|
+
views = configuration.models_module.mdes_order.collect do |model|
|
|
109
|
+
selects = model.properties.collect { |p|
|
|
110
|
+
[p.pii ? "varchar(32) '[pii=#{p.pii}]'" : p.name, p.name].join(' AS ')
|
|
111
|
+
}
|
|
112
|
+
%Q(CREATE VIEW #{no_pii_schema}.#{model.mdes_table_name}
|
|
113
|
+
AS SELECT #{selects.join(',')} FROM public.#{model.mdes_table_name})
|
|
114
|
+
end
|
|
115
|
+
::DataMapper.repository(:mdes_warehouse_working).adapter.
|
|
116
|
+
execute("CREATE SCHEMA #{no_pii_schema} #{views.join("\n")}")
|
|
117
|
+
shell.clear_line_then_say("Created no-PII views.\n")
|
|
118
|
+
|
|
119
|
+
role_name = 'mdes_warehouse_no_pii'
|
|
120
|
+
role_exists = ::DataMapper.repository(:mdes_warehouse_working).adapter.
|
|
121
|
+
select("SELECT rolname FROM pg_catalog.pg_roles WHERE rolname='#{role_name}'").size > 0
|
|
122
|
+
if role_exists
|
|
123
|
+
shell.say("- Granting SELECT on no-PII views to #{role_name}...")
|
|
124
|
+
[
|
|
125
|
+
"GRANT USAGE ON SCHEMA #{no_pii_schema} TO #{role_name}",
|
|
126
|
+
"GRANT SELECT ON ALL TABLES IN SCHEMA #{no_pii_schema} TO #{role_name}"
|
|
127
|
+
].each do |stmt|
|
|
128
|
+
::DataMapper.repository(:mdes_warehouse_working).adapter.execute(stmt)
|
|
129
|
+
end
|
|
130
|
+
shell.clear_line_then_say("- Granted SELECT on no-PII views to #{role_name}.\n")
|
|
131
|
+
else
|
|
132
|
+
shell.say_line(
|
|
133
|
+
"- Database role #{role_name} does not exist. Performing no grants for no-PII views.")
|
|
134
|
+
end
|
|
102
135
|
end
|
|
103
|
-
private :
|
|
136
|
+
private :create_no_pii_views
|
|
104
137
|
|
|
105
138
|
##
|
|
106
139
|
# Replaces the reporting database with a clone of the working
|
|
@@ -117,36 +150,84 @@ module NcsNavigator::Warehouse
|
|
|
117
150
|
pgpass.update params(:reporting)
|
|
118
151
|
end
|
|
119
152
|
|
|
120
|
-
|
|
153
|
+
shell.say 'Cloning working schema into reporting schema...'
|
|
154
|
+
|
|
155
|
+
dump_file = Tempfile.new('wh_clone_dump')
|
|
156
|
+
|
|
157
|
+
dump_cmd = escape_cmd([
|
|
121
158
|
configuration.pg_bin('pg_dump'),
|
|
122
159
|
pg_params(params(:working)),
|
|
123
160
|
'--format=custom',
|
|
161
|
+
"--file=#{dump_file.path}",
|
|
124
162
|
params(:working)['database']
|
|
125
|
-
].flatten
|
|
163
|
+
].flatten)
|
|
164
|
+
|
|
165
|
+
shell.clear_line_then_say(
|
|
166
|
+
"Cloning working to reporting... dumping working database to temporary file")
|
|
167
|
+
log.debug("Dump command: #{dump_cmd}")
|
|
168
|
+
unless system(dump_cmd)
|
|
169
|
+
shell.say(
|
|
170
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
171
|
+
log.error('Dump failed.')
|
|
172
|
+
return false
|
|
173
|
+
end
|
|
126
174
|
|
|
127
|
-
|
|
175
|
+
list_file = Tempfile.new('wh_clone_list')
|
|
176
|
+
list_cmd = escape_cmd([
|
|
177
|
+
configuration.pg_bin('pg_restore'),
|
|
178
|
+
pg_params(params(:working)),
|
|
179
|
+
'--list',
|
|
180
|
+
"--file=#{list_file.path}",
|
|
181
|
+
dump_file.path
|
|
182
|
+
].flatten)
|
|
183
|
+
|
|
184
|
+
shell.clear_line_then_say(
|
|
185
|
+
"Cloning working to reporting... extracting dump contents")
|
|
186
|
+
log.debug("List command: #{list_cmd}")
|
|
187
|
+
unless system(list_cmd)
|
|
188
|
+
shell.say(
|
|
189
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
190
|
+
log.error('Dump failed.')
|
|
191
|
+
return false
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
shell.clear_line_then_say(
|
|
195
|
+
"Cloning working to reporting... filtering content list")
|
|
196
|
+
list_file.rewind
|
|
197
|
+
selection_file = Tempfile.new('wh_clone_selection')
|
|
198
|
+
list_file.read.split("\n").each do |line|
|
|
199
|
+
if line =~ %r{#{params(:working)['username']}$}
|
|
200
|
+
selection_file.puts line
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
restore_cmd = escape_cmd([
|
|
128
205
|
configuration.pg_bin('pg_restore'),
|
|
129
206
|
pg_params(params(:reporting)),
|
|
130
|
-
|
|
131
|
-
'--dbname', params(:reporting)['database']
|
|
132
|
-
|
|
207
|
+
"--use-list=#{selection_file.path}",
|
|
208
|
+
'--dbname', params(:reporting)['database'],
|
|
209
|
+
dump_file.path
|
|
210
|
+
].flatten)
|
|
133
211
|
|
|
134
|
-
drop_all(:reporting)
|
|
212
|
+
drop_all(:reporting, :quiet => true)
|
|
135
213
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
log.
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
shell.clear_line_then_say(
|
|
142
|
-
"Clone from working to reporting failed. See above for detail.\n")
|
|
143
|
-
log.error('Clone failed.')
|
|
144
|
-
return false
|
|
145
|
-
else
|
|
146
|
-
shell.clear_line_then_say("Clone from working to reporting successful.\n")
|
|
214
|
+
shell.clear_line_then_say(
|
|
215
|
+
"Cloning working to reporting... loading filtered dump into reporting database")
|
|
216
|
+
log.debug("Restore command: #{restore_cmd.inspect}")
|
|
217
|
+
if system(restore_cmd)
|
|
218
|
+
shell.clear_line_then_say("Successfully cloned working to reporting.\n")
|
|
147
219
|
log.info('Clone succeeded.')
|
|
148
220
|
return true
|
|
221
|
+
else
|
|
222
|
+
shell.say(
|
|
223
|
+
"\nClone from working to reporting failed. See above for detail.\n")
|
|
224
|
+
log.error('Dump failed.')
|
|
225
|
+
return false
|
|
149
226
|
end
|
|
227
|
+
ensure
|
|
228
|
+
dump_file.unlink if dump_file
|
|
229
|
+
list_file.unlink if list_file
|
|
230
|
+
selection_file.unlink if selection_file
|
|
150
231
|
end
|
|
151
232
|
|
|
152
233
|
def pg_params(p)
|
|
@@ -25,16 +25,22 @@ module NcsNavigator::Warehouse
|
|
|
25
25
|
def repo.identity_map(model); {}; end
|
|
26
26
|
|
|
27
27
|
build_status_for(transformer, position).tap do |status|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
repo.adapter.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
28
|
+
begin
|
|
29
|
+
TransformStatus.transaction do
|
|
30
|
+
if repo.adapter.to_s =~ /Postgres/
|
|
31
|
+
repo.adapter.execute("SET LOCAL synchronous_commit TO OFF")
|
|
32
|
+
end
|
|
33
|
+
begin
|
|
34
|
+
transformer.transform(status)
|
|
35
|
+
rescue => e
|
|
36
|
+
shell.say_line("\nTransform failed. (See log for more detail.)")
|
|
37
|
+
status.add_error("Transform failed. #{e.class}: #{e}.")
|
|
38
|
+
end
|
|
37
39
|
end
|
|
40
|
+
rescue DataObjects::IntegrityError => e
|
|
41
|
+
shell.say_line("\nTransform failed with data integrity error. (See log for more detail.)")
|
|
42
|
+
log.error("Transform failed with data integrity error: #{e}.")
|
|
43
|
+
status.add_error("Transform failed with data integrity error: #{e}.")
|
|
38
44
|
end
|
|
39
45
|
status.end_time = Time.now
|
|
40
46
|
unless status.save
|