loadmop 0.0.4 → 0.0.5

data/lib/loadmop/cli.rb

@@ -6,9 +6,27 @@ module Loadmop
   class CLI < Thor
     include Sequelizer
 
+    class_option :adapter,
+      aliases: :a,
+      desc: 'adapter for database'
+    class_option :host,
+      aliases: :h,
+      banner: 'localhost',
+      desc: 'host for database'
+    class_option :username,
+      aliases: :u,
+      desc: 'username for database'
+    class_option :password,
+      aliases: :P,
+      desc: 'password for database'
+    class_option :port,
+      aliases: :p,
+      type: :numeric,
+      banner: '5432',
+      desc: 'port for database'
     class_option :search_path,
       aliases: :s,
-      desc: 'schema for database (PostgreSQL only)'
+      desc: 'schema for database (PostgreSQL and SQL Server only)'
 
     desc 'create_vocab_database database_name vocab_files_dir', 'Creates the vocabulary tables in database_name and loads the OMOP Vocabulary data into them.'
     def create_vocab_database(database_name, vocab_files_dir)

data/lib/loadmop/loader.rb

@@ -23,13 +23,11 @@ module Loadmop
 
     private
     def create_tables
-      Sequel.extension :migration
-      Sequel::Migrator.run(db, base_dir + schemas_dir, target: 1)
+      raise NotImplementedError
     end
 
     def create_indexes
-      Sequel.extension :migration
-      Sequel::Migrator.run(db, base_dir + schemas_dir, target: 2)
+      raise NotImplementedError
     end
 
     def all_files
@@ -97,13 +95,13 @@ module Loadmop
     end
 
     def slow_load
-      all_files.each do |table_name, files|
+      all_files.each do |table, files|
         files.each do |file|
-          puts "Loading #{file} into #{table_name}"
+          puts "Loading #{file} into #{table}"
           CSV.open(file) do |csv|
             csv.each_slice(1000) do |rows|
               print '.'
-              db[table_name].import(headers_for(file), rows)
+              db[table_name(table)].import(headers_for(file), rows)
             end
           end
           puts
@@ -112,16 +110,16 @@ module Loadmop
     end
 
     def adapter
-      db.sequelizer_options.adapter
+      db.database_type
     end
 
     def database
-      db.sequelizer_options.database
+      db.opts[:database]
     end
 
     def headers_for(file)
       if file.to_s =~ /split/
-        file = Pathname.new(file.dirname.to_s.sub('split', 'cleaned') + '.csv')
+        file = Pathname.new(file.dirname.to_s.sub('split/', '') + '.csv')
       end
       header_line = File.open(file, &:readline).downcase.gsub(/\|$/, '')
       CSV.parse(header_line).first.map(&:to_sym)
@@ -163,11 +161,40 @@ module Loadmop
       Hash[files]
     end
 
-    # When installed as a gem, we need to find where the gem is installed
-    # and look for files relative to that path.  base_dir returns the
-    # path to the loadmop gem
-    def base_dir
-      Pathname.new(__FILE__).dirname + '../..'
+    def schemas
+      @schemas ||= (options[:search_path] || '').split(',').map(&:strip).map(&:to_sym)
+    end
+
+    def table_name(name)
+      return name unless schemas.length > 0
+      [schemas.first, name].map(&:to_s).join('__').to_sym
+    end
+
+    def create_schema_if_necessary
+      return unless options[:search_path]
+      if db.database_type == :mssql
+        schemas.each do |schema|
+          schema = schema.to_s.upcase
+
+          create_if_not_exists = <<-EOF
+          IF NOT EXISTS (
+          SELECT  name
+          FROM    sys.schemas
+          WHERE   name = '#{schema}' )
+
+          BEGIN
+          EXEC sp_executesql N'CREATE SCHEMA #{schema}'
+          END
+          EOF
+
+          db.execute(create_if_not_exists)
+        end
+      elsif db.database_type == :postgres
+        schemas.each do |schema|
+          db.execute("CREATE SCHEMA IF NOT EXISTS #{schema}")
+        end
+        db.execute("SET search_path TO #{options[:search_path]}")
+      end
     end
   end
 end

data/lib/loadmop/version.rb

@@ -1,3 +1,3 @@
 module Loadmop
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/loadmop/vocab_loader.rb

@@ -4,13 +4,125 @@ module Loadmop
   class VocabLoader < Loader
 
     private
+    def additional_cleaning_steps
+      ["sed 's/\|$//'"]
+    end
+
+    def create_tables
+      create_schema_if_necessary
+      db.create_table!(table_name(:concept)) do
+        Bignum :concept_id, :null=>false
+        String :concept_name, :null=>false
+        BigDecimal :concept_level, :null=>false
+        String :concept_class, :null=>false
+        Bignum :vocabulary_id, :null=>false
+        String :concept_code, :null=>false
+        Date :valid_start_date, :null=>false
+        Date :valid_end_date, :null=>false
+        String :invalid_reason, :size=>1, :fixed=>true
+
+        primary_key [:concept_id]
+      end
+
+      db.create_table!(table_name(:concept_ancestor)) do
+        Bignum :ancestor_concept_id, :null=>false
+        Bignum :descendant_concept_id, :null=>false
+        BigDecimal :min_levels_of_separation
+        BigDecimal :max_levels_of_separation
+
+        primary_key [:ancestor_concept_id, :descendant_concept_id]
+      end
+
+      db.create_table!(table_name(:concept_relationship)) do
+        Bignum :concept_id_1, :null=>false
+        Bignum :concept_id_2, :null=>false
+        Bignum :relationship_id, :null=>false
+        Date :valid_start_date, :null=>false
+        Date :valid_end_date, :null=>false
+        String :invalid_reason, :size=>1, :fixed=>true
+
+        primary_key [:concept_id_1, :concept_id_2, :relationship_id]
+      end
+
+      db.create_table!(table_name(:concept_synonym)) do
+        Bignum :concept_synonym_id, :null=>false
+        Bignum :concept_id, :null=>false
+        String :concept_synonym_name, :null=>false
+
+        primary_key [:concept_synonym_id]
+      end
 
-    def schemas_dir
-      'schemas/vocabulary'
+      db.create_table!(table_name(:drug_approval)) do
+        Bignum :ingredient_concept_id, :null => false
+        Date :approval_date, :null => false
+        String :approved_by, :null => false
+      end
+
+      db.create_table!(table_name(:drug_strength)) do
+        Bignum :drug_concept_id, :null => false
+        Bignum :ingredient_concept_id, :null => false
+        BigDecimal :amount_value
+        String :amount_unit
+        BigDecimal :concentration_value
+        String :concentration_enum_unit
+        String :concentration_denom_unit
+        Date :valid_start_date, :null => false
+        Date :valid_end_date, :null => false
+        String :invalid_reason
+      end
+
+      db.create_table!(table_name(:relationship)) do
+        Bignum :relationship_id, :null=>false
+        String :relationship_name, :null=>false
+        String :is_hierarchical, :size=>1, :fixed=>true
+        String :defines_ancestry, :size=>1, :fixed=>true
+        Bignum :reverse_relationship
+
+        primary_key [:relationship_id]
+      end
+
+      db.create_table!(table_name(:source_to_concept_map)) do
+        String :source_code, :null=>false
+        Bignum :source_vocabulary_id, :null=>false
+        String :source_code_description
+        Bignum :target_concept_id, :null=>false
+        Bignum :target_vocabulary_id, :null=>false
+        String :mapping_type
+        String :primary_map, :size=>1, :fixed=>true
+        Date :valid_start_date, :null=>false
+        Date :valid_end_date, :null=>false
+        String :invalid_reason, :size=>1, :fixed=>true
+
+        primary_key [:source_code, :source_vocabulary_id, :target_concept_id, :valid_end_date]
+      end
+
+      db.create_table!(table_name(:vocabulary)) do
+        Bignum :vocabulary_id, :null=>false
+        String :vocabulary_name, :null=>false
+
+        primary_key [:vocabulary_id]
+      end
     end
 
-    def additional_cleaning_steps
-      ["sed 's/\|$//'"]
+    def do_index(*args)
+      db.add_index(*args)
+    rescue
+      puts $!.message
+      #nothing
+    end
+
+    def create_indexes
+      do_index :concept, [:concept_code], name: "vocabulary_concept_concept_code_index"
+      do_index :concept, [:concept_id], name: "con_conid"
+      do_index :concept, [:vocabulary_id], name: "con_vocid"
+      do_index :concept_ancestor, [:ancestor_concept_id], name: "conanc_ancconid"
+      do_index :concept_ancestor, [:descendant_concept_id], name: "conanc_desconid"
+      do_index :concept_relationship, [:relationship_id], name: "conrel_relid"
+      do_index :relationship, [:relationship_id], name: "rel_relid"
+      do_index :source_to_concept_map, [:source_code, :source_vocabulary_id], name: "soutoconmap_soucod_souvocid"
+      do_index :source_to_concept_map, [:source_vocabulary_id], name: "soutoconmap_souvocid"
+      do_index :source_to_concept_map, [:target_concept_id], name: "soutoconmap_tarconid"
+      do_index :source_to_concept_map, [:target_vocabulary_id], name: "soutoconmap_tarvocid"
     end
   end
 end

data/loadmop.gemspec

@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_dependency 'rake', '~> 10.3'
-  spec.add_dependency 'sequelizer', '~> 0.0'
+  spec.add_dependency 'sequelizer', '>= 0.0.5'
   spec.add_dependency 'thor', '~> 0.19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: loadmop
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ryan Duryea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-29 00:00:00.000000000 Z
+date: 2014-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,16 +42,16 @@ dependencies:
   name: sequelizer
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: 0.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: 0.0.5
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -90,10 +90,6 @@ files:
 - lib/loadmop/version.rb
 - lib/loadmop/vocab_loader.rb
 - loadmop.gemspec
-- schemas/cdmv4/001_tables.rb
-- schemas/cdmv4/002_indexes.rb
-- schemas/vocabulary/001_tables.rb
-- schemas/vocabulary/002_indexes.rb
 homepage: https://github.com/outcomesinsights/loadmop
 licenses:
 - MIT

data/schemas/cdmv4/001_tables.rb

@@ -1,256 +0,0 @@
-Sequel.migration do
-  change do
-    create_table(:care_site, :ignore_index_errors=>true) do
-      Bignum :care_site_id, :null=>false
-      Bignum :location_id, :null=>false
-      Bignum :organization_id, :null=>false
-      Bignum :place_of_service_concept_id
-      String :care_site_source_value, :size=>50
-      String :place_of_service_source_value, :size=>50, :null=>false
-
-      primary_key [:care_site_id]
-    end
-
-    create_table(:cohort, :ignore_index_errors=>true) do
-      Bignum :cohort_id, :null=>false
-      Bignum :cohort_concept_id, :null=>false
-      Date :cohort_start_date, :null=>false
-      Date :cohort_end_date
-      Bignum :subject_id, :null=>false
-      String :stop_reason, :size=>20
-
-      primary_key [:cohort_id]
-    end
-
-    create_table(:location, :ignore_index_errors=>true) do
-      Bignum :location_id, :null=>false
-      String :address_1, :size=>50
-      String :address_2, :size=>50
-      String :city, :size=>50
-      String :state, :size=>2, :fixed=>true
-      String :zip, :size=>9
-      String :county, :size=>20
-      String :location_source_value, :size=>50
-
-      primary_key [:location_id]
-    end
-
-    create_table(:provider, :ignore_index_errors=>true) do
-      Bignum :provider_id, :null=>false
-      String :npi, :size=>20
-      String :dea, :size=>20
-      Bignum :specialty_concept_id
-      Bignum :care_site_id, :null=>false
-      String :provider_source_value, :size=>50, :null=>false
-      String :specialty_source_value, :size=>50
-
-      primary_key [:provider_id]
-    end
-
-    create_table(:organization, :ignore_index_errors=>true) do
-      Bignum :organization_id, :null=>false
-      Bignum :place_of_service_concept_id
-      foreign_key :location_id, :location, :type=>Bignum, :key=>[:location_id]
-      String :organization_source_value, :size=>50, :null=>false
-      String :place_of_service_source_value, :size=>50
-
-      primary_key [:organization_id]
-    end
-
-    create_table(:person, :ignore_index_errors=>true) do
-      Bignum :person_id, :null=>false
-      Bignum :gender_concept_id, :null=>false
-      Integer :year_of_birth, :null=>false
-      Integer :month_of_birth
-      Integer :day_of_birth
-      Bignum :race_concept_id
-      Bignum :ethnicity_concept_id
-      foreign_key :location_id, :location, :type=>Bignum, :key=>[:location_id]
-      foreign_key :provider_id, :provider, :type=>Bignum, :key=>[:provider_id]
-      Bignum :care_site_id
-      String :person_source_value, :size=>50
-      String :gender_source_value, :size=>50
-      String :race_source_value, :size=>50
-      String :ethnicity_source_value, :size=>50
-
-      primary_key [:person_id]
-    end
-
-    create_table(:condition_era, :ignore_index_errors=>true) do
-      Bignum :condition_era_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :condition_concept_id, :null=>false
-      Date :condition_era_start_date, :null=>false
-      Date :condition_era_end_date, :null=>false
-      Bignum :condition_type_concept_id, :null=>false
-      Integer :condition_occurrence_count
-
-      primary_key [:condition_era_id]
-    end
-
-    create_table(:condition_occurrence, :ignore_index_errors=>true) do
-      Bignum :condition_occurrence_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :condition_concept_id, :null=>false
-      Date :condition_start_date, :null=>false
-      Date :condition_end_date
-      Bignum :condition_type_concept_id, :null=>false
-      String :stop_reason, :size=>20
-      Bignum :associated_provider_id
-      Bignum :visit_occurrence_id
-      String :condition_source_value, :size=>50
-
-      primary_key [:condition_occurrence_id]
-    end
-
-    create_table(:death, :ignore_index_errors=>true) do
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Date :death_date, :null=>false
-      Bignum :death_type_concept_id, :null=>false
-      Bignum :cause_of_death_concept_id
-      String :cause_of_death_source_value, :size=>50
-
-      primary_key [:person_id]
-    end
-
-    create_table(:drug_era, :ignore_index_errors=>true) do
-      Bignum :drug_era_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :drug_concept_id, :null=>false
-      Date :drug_era_start_date, :null=>false
-      Date :drug_era_end_date, :null=>false
-      Bignum :drug_type_concept_id, :null=>false
-      Integer :drug_exposure_count
-
-      primary_key [:drug_era_id]
-    end
-
-    create_table(:drug_exposure, :ignore_index_errors=>true) do
-      Bignum :drug_exposure_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :drug_concept_id, :null=>false
-      Date :drug_exposure_start_date, :null=>false
-      Date :drug_exposure_end_date
-      Bignum :drug_type_concept_id, :null=>false
-      String :stop_reason, :size=>20
-      Integer :refills
-      Integer :quantity
-      Integer :days_supply
-      String :sig, :size=>500
-      Bignum :prescribing_provider_id
-      Bignum :visit_occurrence_id
-      Bignum :relevant_condition_concept_id
-      String :drug_source_value, :size=>50
-
-      primary_key [:drug_exposure_id]
-    end
-
-    create_table(:observation, :ignore_index_errors=>true) do
-      Bignum :observation_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :observation_concept_id, :null=>false
-      Date :observation_date, :null=>false
-      Date :observation_time
-      Float :value_as_number
-      String :value_as_string, :size=>60
-      Bignum :value_as_concept_id
-      Bignum :unit_concept_id
-      Float :range_low
-      Float :range_high
-      Bignum :observation_type_concept_id, :null=>false
-      Bignum :associated_provider_id
-      Bignum :visit_occurrence_id
-      Bignum :relevant_condition_concept_id
-      String :observation_source_value, :size=>50
-      String :units_source_value, :size=>50
-
-      primary_key [:observation_id]
-    end
-
-    create_table(:observation_period, :ignore_index_errors=>true) do
-      Bignum :observation_period_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Date :observation_period_start_date, :null=>false
-      Date :observation_period_end_date, :null=>false
-      Date :prev_ds_period_end_date
-
-      primary_key [:observation_period_id]
-    end
-
-    create_table(:payer_plan_period, :ignore_index_errors=>true) do
-      Bignum :payer_plan_period_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Date :payer_plan_period_start_date, :null=>false
-      Date :payer_plan_period_end_date, :null=>false
-      String :payer_source_value, :size=>50
-      String :plan_source_value, :size=>50
-      String :family_source_value, :size=>50
-      Date :prev_ds_period_end_date
-
-      primary_key [:payer_plan_period_id]
-    end
-
-    create_table(:procedure_occurrence, :ignore_index_errors=>true) do
-      Bignum :procedure_occurrence_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Bignum :procedure_concept_id, :null=>false
-      Date :procedure_date, :null=>false
-      Bignum :procedure_type_concept_id, :null=>false
-      Bignum :associated_provider_id
-      Bignum :visit_occurrence_id
-      Bignum :relevant_condition_concept_id
-      String :procedure_source_value, :size=>50
-
-      primary_key [:procedure_occurrence_id]
-    end
-
-    create_table(:visit_occurrence, :ignore_index_errors=>true) do
-      Bignum :visit_occurrence_id, :null=>false
-      foreign_key :person_id, :person, :type=>Bignum, :null=>false, :key=>[:person_id]
-      Date :visit_start_date, :null=>false
-      Date :visit_end_date, :null=>false
-      Bignum :place_of_service_concept_id, :null=>false
-      Bignum :care_site_id
-      String :place_of_service_source_value, :size=>50
-
-      primary_key [:visit_occurrence_id]
-    end
-
-    create_table(:drug_cost, :ignore_index_errors=>true) do
-      Bignum :drug_cost_id, :null=>false
-      foreign_key :drug_exposure_id, :drug_exposure, :type=>Bignum, :null=>false, :key=>[:drug_exposure_id]
-      Float :paid_copay
-      Float :paid_coinsurance
-      Float :paid_toward_deductible
-      Float :paid_by_payer
-      Float :paid_by_coordination_benefits
-      Float :total_out_of_pocket
-      Float :total_paid
-      Float :ingredient_cost
-      Float :dispensing_fee
-      Float :average_wholesale_price
-      Bignum :payer_plan_period_id
-
-      primary_key [:drug_cost_id]
-    end
-
-    create_table(:procedure_cost, :ignore_index_errors=>true) do
-      Bignum :procedure_cost_id, :null=>false
-      foreign_key :procedure_occurrence_id, :procedure_occurrence, :type=>Bignum, :null=>false, :key=>[:procedure_occurrence_id]
-      Float :paid_copay
-      Float :paid_coinsurance
-      Float :paid_toward_deductible
-      Float :paid_by_payer
-      Float :paid_by_coordination_benefits
-      Float :total_out_of_pocket
-      Float :total_paid
-      Bignum :disease_class_concept_id
-      Bignum :revenue_code_concept_id
-      Bignum :payer_plan_period_id
-      String :disease_class_source_value, :size=>50
-      String :revenue_code_source_value, :size=>50
-
-      primary_key [:procedure_cost_id]
-    end
-  end
-end