loadmop 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,9 +6,27 @@ module Loadmop
6
6
  class CLI < Thor
7
7
  include Sequelizer
8
8
 
9
+ class_option :adapter,
10
+ aliases: :a,
11
+ desc: 'adapter for database'
12
+ class_option :host,
13
+ aliases: :h,
14
+ banner: 'localhost',
15
+ desc: 'host for database'
16
+ class_option :username,
17
+ aliases: :u,
18
+ desc: 'username for database'
19
+ class_option :password,
20
+ aliases: :P,
21
+ desc: 'password for database'
22
+ class_option :port,
23
+ aliases: :p,
24
+ type: :numeric,
25
+ banner: '5432',
26
+ desc: 'port for database'
9
27
  class_option :search_path,
10
28
  aliases: :s,
11
- desc: 'schema for database (PostgreSQL only)'
29
+ desc: 'schema for database (PostgreSQL and SQL Server only)'
12
30
 
13
31
  desc 'create_vocab_database database_name vocab_files_dir', 'Creates the vocabulary tables in database_name and loads the OMOP Vocabulary data into them.'
14
32
  def create_vocab_database(database_name, vocab_files_dir)
@@ -23,13 +23,11 @@ module Loadmop
23
23
 
24
24
  private
25
25
  def create_tables
26
- Sequel.extension :migration
27
- Sequel::Migrator.run(db, base_dir + schemas_dir, target: 1)
26
+ raise NotImplementedError
28
27
  end
29
28
 
30
29
  def create_indexes
31
- Sequel.extension :migration
32
- Sequel::Migrator.run(db, base_dir + schemas_dir, target: 2)
30
+ raise NotImplementedError
33
31
  end
34
32
 
35
33
  def all_files
@@ -97,13 +95,13 @@ module Loadmop
97
95
  end
98
96
 
99
97
  def slow_load
100
- all_files.each do |table_name, files|
98
+ all_files.each do |table, files|
101
99
  files.each do |file|
102
- puts "Loading #{file} into #{table_name}"
100
+ puts "Loading #{file} into #{table}"
103
101
  CSV.open(file) do |csv|
104
102
  csv.each_slice(1000) do |rows|
105
103
  print '.'
106
- db[table_name].import(headers_for(file), rows)
104
+ db[table_name(table)].import(headers_for(file), rows)
107
105
  end
108
106
  end
109
107
  puts
@@ -112,16 +110,16 @@ module Loadmop
112
110
  end
113
111
 
114
112
  def adapter
115
- db.sequelizer_options.adapter
113
+ db.database_type
116
114
  end
117
115
 
118
116
  def database
119
- db.sequelizer_options.database
117
+ db.opts[:database]
120
118
  end
121
119
 
122
120
  def headers_for(file)
123
121
  if file.to_s =~ /split/
124
- file = Pathname.new(file.dirname.to_s.sub('split', 'cleaned') + '.csv')
122
+ file = Pathname.new(file.dirname.to_s.sub('split/', '') + '.csv')
125
123
  end
126
124
  header_line = File.open(file, &:readline).downcase.gsub(/\|$/, '')
127
125
  CSV.parse(header_line).first.map(&:to_sym)
@@ -163,11 +161,40 @@ module Loadmop
163
161
  Hash[files]
164
162
  end
165
163
 
166
- # When installed as a gem, we need to find where the gem is installed
167
- # and look for files relative to that path. base_dir returns the
168
- # path to the loadmop gem
169
- def base_dir
170
- Pathname.new(__FILE__).dirname + '../..'
164
+ def schemas
165
+ @schemas ||= (options[:search_path] || '').split(',').map(&:strip).map(&:to_sym)
166
+ end
167
+
168
+ def table_name(name)
169
+ return name unless schemas.length > 0
170
+ [schemas.first, name].map(&:to_s).join('__').to_sym
171
+ end
172
+
173
+ def create_schema_if_necessary
174
+ return unless options[:search_path]
175
+ if db.database_type == :mssql
176
+ schemas.each do |schema|
177
+ schema = schema.to_s.upcase
178
+
179
+ create_if_not_exists = <<-EOF
180
+ IF NOT EXISTS (
181
+ SELECT name
182
+ FROM sys.schemas
183
+ WHERE name = '#{schema}' )
184
+
185
+ BEGIN
186
+ EXEC sp_executesql N'CREATE SCHEMA #{schema}'
187
+ END
188
+ EOF
189
+
190
+ db.execute(create_if_not_exists)
191
+ end
192
+ elsif db.database_type == :postgres
193
+ schemas.each do |schema|
194
+ db.execute("CREATE SCHEMA IF NOT EXISTS #{schema}")
195
+ end
196
+ db.execute("SET search_path TO #{options[:search_path]}")
197
+ end
171
198
  end
172
199
  end
173
200
  end
@@ -1,3 +1,3 @@
1
1
  module Loadmop
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.5"
3
3
  end
@@ -4,13 +4,125 @@ module Loadmop
4
4
  class VocabLoader < Loader
5
5
 
6
6
  private
7
+ def additional_cleaning_steps
8
+ ["sed 's/\|$//'"]
9
+ end
10
+
11
+ def create_tables
12
+ create_schema_if_necessary
13
+ db.create_table!(table_name(:concept)) do
14
+ Bignum :concept_id, :null=>false
15
+ String :concept_name, :null=>false
16
+ BigDecimal :concept_level, :null=>false
17
+ String :concept_class, :null=>false
18
+ Bignum :vocabulary_id, :null=>false
19
+ String :concept_code, :null=>false
20
+ Date :valid_start_date, :null=>false
21
+ Date :valid_end_date, :null=>false
22
+ String :invalid_reason, :size=>1, :fixed=>true
23
+
24
+ primary_key [:concept_id]
25
+ end
26
+
27
+ db.create_table!(table_name(:concept_ancestor)) do
28
+ Bignum :ancestor_concept_id, :null=>false
29
+ Bignum :descendant_concept_id, :null=>false
30
+ BigDecimal :min_levels_of_separation
31
+ BigDecimal :max_levels_of_separation
32
+
33
+ primary_key [:ancestor_concept_id, :descendant_concept_id]
34
+ end
35
+
36
+ db.create_table!(table_name(:concept_relationship)) do
37
+ Bignum :concept_id_1, :null=>false
38
+ Bignum :concept_id_2, :null=>false
39
+ Bignum :relationship_id, :null=>false
40
+ Date :valid_start_date, :null=>false
41
+ Date :valid_end_date, :null=>false
42
+ String :invalid_reason, :size=>1, :fixed=>true
43
+
44
+ primary_key [:concept_id_1, :concept_id_2, :relationship_id]
45
+ end
46
+
47
+ db.create_table!(table_name(:concept_synonym)) do
48
+ Bignum :concept_synonym_id, :null=>false
49
+ Bignum :concept_id, :null=>false
50
+ String :concept_synonym_name, :null=>false
51
+
52
+ primary_key [:concept_synonym_id]
53
+ end
7
54
 
8
- def schemas_dir
9
- 'schemas/vocabulary'
55
+ db.create_table!(table_name(:drug_approval)) do
56
+ Bignum :ingredient_concept_id, :null => false
57
+ Date :approval_date, :null => false
58
+ String :approved_by, :null => false
59
+ end
60
+
61
+ db.create_table!(table_name(:drug_strength)) do
62
+ Bignum :drug_concept_id, :null => false
63
+ Bignum :ingredient_concept_id, :null => false
64
+ BigDecimal :amount_value
65
+ String :amount_unit
66
+ BigDecimal :concentration_value
67
+ String :concentration_enum_unit
68
+ String :concentration_denom_unit
69
+ Date :valid_start_date, :null => false
70
+ Date :valid_end_date, :null => false
71
+ String :invalid_reason
72
+ end
73
+
74
+ db.create_table!(table_name(:relationship)) do
75
+ Bignum :relationship_id, :null=>false
76
+ String :relationship_name, :null=>false
77
+ String :is_hierarchical, :size=>1, :fixed=>true
78
+ String :defines_ancestry, :size=>1, :fixed=>true
79
+ Bignum :reverse_relationship
80
+
81
+ primary_key [:relationship_id]
82
+ end
83
+
84
+ db.create_table!(table_name(:source_to_concept_map)) do
85
+ String :source_code, :null=>false
86
+ Bignum :source_vocabulary_id, :null=>false
87
+ String :source_code_description
88
+ Bignum :target_concept_id, :null=>false
89
+ Bignum :target_vocabulary_id, :null=>false
90
+ String :mapping_type
91
+ String :primary_map, :size=>1, :fixed=>true
92
+ Date :valid_start_date, :null=>false
93
+ Date :valid_end_date, :null=>false
94
+ String :invalid_reason, :size=>1, :fixed=>true
95
+
96
+ primary_key [:source_code, :source_vocabulary_id, :target_concept_id, :valid_end_date]
97
+ end
98
+
99
+ db.create_table!(table_name(:vocabulary)) do
100
+ Bignum :vocabulary_id, :null=>false
101
+ String :vocabulary_name, :null=>false
102
+
103
+ primary_key [:vocabulary_id]
104
+ end
10
105
  end
11
106
 
12
- def additional_cleaning_steps
13
- ["sed 's/\|$//'"]
107
+ def do_index(*args)
108
+ db.add_index(*args)
109
+ rescue
110
+ puts $!.message
111
+ #nothing
112
+ end
113
+
114
+ def create_indexes
115
+ do_index :concept, [:concept_code], name: "vocabulary_concept_concept_code_index"
116
+ do_index :concept, [:concept_id], name: "con_conid"
117
+ do_index :concept, [:vocabulary_id], name: "con_vocid"
118
+ do_index :concept_ancestor, [:ancestor_concept_id], name: "conanc_ancconid"
119
+ do_index :concept_ancestor, [:descendant_concept_id], name: "conanc_desconid"
120
+ do_index :concept_relationship, [:relationship_id], name: "conrel_relid"
121
+ do_index :relationship, [:relationship_id], name: "rel_relid"
122
+ do_index :source_to_concept_map, [:source_code, :source_vocabulary_id], name: "soutoconmap_soucod_souvocid"
123
+ do_index :source_to_concept_map, [:source_vocabulary_id], name: "soutoconmap_souvocid"
124
+ do_index :source_to_concept_map, [:target_concept_id], name: "soutoconmap_tarconid"
125
+ do_index :source_to_concept_map, [:target_vocabulary_id], name: "soutoconmap_tarvocid"
14
126
  end
15
127
  end
16
128
  end
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.add_development_dependency 'bundler', '~> 1.6'
22
22
  spec.add_dependency 'rake', '~> 10.3'
23
- spec.add_dependency 'sequelizer', '~> 0.0'
23
+ spec.add_dependency 'sequelizer', '>= 0.0.5'
24
24
  spec.add_dependency 'thor', '~> 0.19'
25
25
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: loadmop
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Duryea
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-08-29 00:00:00.000000000 Z
11
+ date: 2014-09-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -42,16 +42,16 @@ dependencies:
42
42
  name: sequelizer
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: '0.0'
47
+ version: 0.0.5
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: '0.0'
54
+ version: 0.0.5
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: thor
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -90,10 +90,6 @@ files:
90
90
  - lib/loadmop/version.rb
91
91
  - lib/loadmop/vocab_loader.rb
92
92
  - loadmop.gemspec
93
- - schemas/cdmv4/001_tables.rb
94
- - schemas/cdmv4/002_indexes.rb
95
- - schemas/vocabulary/001_tables.rb
96
- - schemas/vocabulary/002_indexes.rb
97
93
  homepage: https://github.com/outcomesinsights/loadmop
98
94
  licenses:
99
95
  - MIT
@@ -1,256 +0,0 @@
1
- Sequel.migration do
2
- change do
3
- create_table(:care_site, :ignore_index_errors=>true) do
4
- Bignum :care_site_id, :null=>false
5
- Bignum :location_id, :null=>false
6
- Bignum :organization_id, :null=>false
7
- Bignum :place_of_service_concept_id
8
- String :care_site_source_value, :size=>50
9
- String :place_of_service_source_value, :size=>50, :null=>false
10
-
11
- primary_key [:care_site_id]
12
- end
13
-
14
- create_table(:cohort, :ignore_index_errors=>true) do
15
- Bignum :cohort_id, :null=>false
16
- Bignum :cohort_concept_id, :null=>false
17
- Date :cohort_start_date, :null=>false
18
- Date :cohort_end_date
19
- Bignum :subject_id, :null=>false
20
- String :stop_reason, :size=>20
21
-
22
- primary_key [:cohort_id]
23
- end
24
-
25
- create_table(:location, :ignore_index_errors=>true) do
26
- Bignum :location_id, :null=>false
27
- String :address_1, :size=>50
28
- String :address_2, :size=>50
29
- String :city, :size=>50
30
- String :state, :size=>2, :fixed=>true
31
- String :zip, :size=>9
32
- String :county, :size=>20
33
- String :location_source_value, :size=>50
34
-
35
- primary_key [:location_id]
36
- end
37
-
38
- create_table(:provider, :ignore_index_errors=>true) do
39
- Bignum :provider_id, :null=>false
40
- String :npi, :size=>20
41
- String :dea, :size=>20
42
- Bignum :specialty_concept_id
43
- Bignum :care_site_id, :null=>false
44
- String :provider_source_value, :size=>50, :null=>false
45
- String :specialty_source_value, :size=>50
46
-
47
- primary_key [:provider_id]
48
- end
49
-
50
- create_table(:organization, :ignore_index_errors=>true) do
51
- Bignum :organization_id, :null=>false
52
- Bignum :place_of_service_concept_id
53
- foreign_key :location_id, :location, :type=>Bignum, :key=>[:location_id]
54
- String :organization_source_value, :size=>50, :null=>false
55
- String :place_of_service_source_value, :size=>50
56
-
57
- primary_key [:organization_id]
58
- end
59
-
60
- create_table(:person, :ignore_index_errors=>true) do
61
- Bignum :person_id, :null=>false
62
- Bignum :gender_concept_id, :null=>false
63
- Integer :year_of_birth, :null=>false
64
- Integer :month_of_birth
65
- Integer :day_of_birth
66
- Bignum :race_concept_id
67
- Bignum :ethnicity_concept_id
68
- foreign_key :location_id, :location, :type=>Bignum, :key=>[:location_id]
69
- foreign_key :provider_id, :provider, :type=>Bignum, :key=>[:provider_id]
70
- Bignum :care_site_id
71
- String :person_source_value, :size=>50
72
- String :gender_source_value, :size=>50
73
- String :race_source_value, :size=>50
74
- String :ethnicity_source_value, :size=>50
75
-
76
- primary_key [:person_id]
77
- end
78
-
79
- create_table(:condition_era, :ignore_index_errors=>true) do
80
- Bignum :condition_era_id, :null=>false
81
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
82
- Bignum :condition_concept_id, :null=>false
83
- Date :condition_era_start_date, :null=>false
84
- Date :condition_era_end_date, :null=>false
85
- Bignum :condition_type_concept_id, :null=>false
86
- Integer :condition_occurrence_count
87
-
88
- primary_key [:condition_era_id]
89
- end
90
-
91
- create_table(:condition_occurrence, :ignore_index_errors=>true) do
92
- Bignum :condition_occurrence_id, :null=>false
93
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
94
- Bignum :condition_concept_id, :null=>false
95
- Date :condition_start_date, :null=>false
96
- Date :condition_end_date
97
- Bignum :condition_type_concept_id, :null=>false
98
- String :stop_reason, :size=>20
99
- Bignum :associated_provider_id
100
- Bignum :visit_occurrence_id
101
- String :condition_source_value, :size=>50
102
-
103
- primary_key [:condition_occurrence_id]
104
- end
105
-
106
- create_table(:death, :ignore_index_errors=>true) do
107
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
108
- Date :death_date, :null=>false
109
- Bignum :death_type_concept_id, :null=>false
110
- Bignum :cause_of_death_concept_id
111
- String :cause_of_death_source_value, :size=>50
112
-
113
- primary_key [:person_id]
114
- end
115
-
116
- create_table(:drug_era, :ignore_index_errors=>true) do
117
- Bignum :drug_era_id, :null=>false
118
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
119
- Bignum :drug_concept_id, :null=>false
120
- Date :drug_era_start_date, :null=>false
121
- Date :drug_era_end_date, :null=>false
122
- Bignum :drug_type_concept_id, :null=>false
123
- Integer :drug_exposure_count
124
-
125
- primary_key [:drug_era_id]
126
- end
127
-
128
- create_table(:drug_exposure, :ignore_index_errors=>true) do
129
- Bignum :drug_exposure_id, :null=>false
130
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
131
- Bignum :drug_concept_id, :null=>false
132
- Date :drug_exposure_start_date, :null=>false
133
- Date :drug_exposure_end_date
134
- Bignum :drug_type_concept_id, :null=>false
135
- String :stop_reason, :size=>20
136
- Integer :refills
137
- Integer :quantity
138
- Integer :days_supply
139
- String :sig, :size=>500
140
- Bignum :prescribing_provider_id
141
- Bignum :visit_occurrence_id
142
- Bignum :relevant_condition_concept_id
143
- String :drug_source_value, :size=>50
144
-
145
- primary_key [:drug_exposure_id]
146
- end
147
-
148
- create_table(:observation, :ignore_index_errors=>true) do
149
- Bignum :observation_id, :null=>false
150
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
151
- Bignum :observation_concept_id, :null=>false
152
- Date :observation_date, :null=>false
153
- Date :observation_time
154
- Float :value_as_number
155
- String :value_as_string, :size=>60
156
- Bignum :value_as_concept_id
157
- Bignum :unit_concept_id
158
- Float :range_low
159
- Float :range_high
160
- Bignum :observation_type_concept_id, :null=>false
161
- Bignum :associated_provider_id
162
- Bignum :visit_occurrence_id
163
- Bignum :relevant_condition_concept_id
164
- String :observation_source_value, :size=>50
165
- String :units_source_value, :size=>50
166
-
167
- primary_key [:observation_id]
168
- end
169
-
170
- create_table(:observation_period, :ignore_index_errors=>true) do
171
- Bignum :observation_period_id, :null=>false
172
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
173
- Date :observation_period_start_date, :null=>false
174
- Date :observation_period_end_date, :null=>false
175
- Date :prev_ds_period_end_date
176
-
177
- primary_key [:observation_period_id]
178
- end
179
-
180
- create_table(:payer_plan_period, :ignore_index_errors=>true) do
181
- Bignum :payer_plan_period_id, :null=>false
182
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
183
- Date :payer_plan_period_start_date, :null=>false
184
- Date :payer_plan_period_end_date, :null=>false
185
- String :payer_source_value, :size=>50
186
- String :plan_source_value, :size=>50
187
- String :family_source_value, :size=>50
188
- Date :prev_ds_period_end_date
189
-
190
- primary_key [:payer_plan_period_id]
191
- end
192
-
193
- create_table(:procedure_occurrence, :ignore_index_errors=>true) do
194
- Bignum :procedure_occurrence_id, :null=>false
195
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
196
- Bignum :procedure_concept_id, :null=>false
197
- Date :procedure_date, :null=>false
198
- Bignum :procedure_type_concept_id, :null=>false
199
- Bignum :associated_provider_id
200
- Bignum :visit_occurrence_id
201
- Bignum :relevant_condition_concept_id
202
- String :procedure_source_value, :size=>50
203
-
204
- primary_key [:procedure_occurrence_id]
205
- end
206
-
207
- create_table(:visit_occurrence, :ignore_index_errors=>true) do
208
- Bignum :visit_occurrence_id, :null=>false
209
- foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
210
- Date :visit_start_date, :null=>false
211
- Date :visit_end_date, :null=>false
212
- Bignum :place_of_service_concept_id, :null=>false
213
- Bignum :care_site_id
214
- String :place_of_service_source_value, :size=>50
215
-
216
- primary_key [:visit_occurrence_id]
217
- end
218
-
219
- create_table(:drug_cost, :ignore_index_errors=>true) do
220
- Bignum :drug_cost_id, :null=>false
221
- foreign_key :drug_exposure_id, :drug_exposure, :type=>Bignum, :null=>false, :key=>[:drug_exposure_id]
222
- Float :paid_copay
223
- Float :paid_coinsurance
224
- Float :paid_toward_deductible
225
- Float :paid_by_payer
226
- Float :paid_by_coordination_benefits
227
- Float :total_out_of_pocket
228
- Float :total_paid
229
- Float :ingredient_cost
230
- Float :dispensing_fee
231
- Float :average_wholesale_price
232
- Bignum :payer_plan_period_id
233
-
234
- primary_key [:drug_cost_id]
235
- end
236
-
237
- create_table(:procedure_cost, :ignore_index_errors=>true) do
238
- Bignum :procedure_cost_id, :null=>false
239
- foreign_key :procedure_occurrence_id, :procedure_occurrence, :type=>Bignum, :null=>false, :key=>[:procedure_occurrence_id]
240
- Float :paid_copay
241
- Float :paid_coinsurance
242
- Float :paid_toward_deductible
243
- Float :paid_by_payer
244
- Float :paid_by_coordination_benefits
245
- Float :total_out_of_pocket
246
- Float :total_paid
247
- Bignum :disease_class_concept_id
248
- Bignum :revenue_code_concept_id
249
- Bignum :payer_plan_period_id
250
- String :disease_class_source_value, :size=>50
251
- String :revenue_code_source_value, :size=>50
252
-
253
- primary_key [:procedure_cost_id]
254
- end
255
- end
256
- end