conceptql 0.2.0 → 0.4.0

data/lib/conceptql/data_model/base.rb

@@ -0,0 +1,462 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "psych"
+
+module ConceptQL
+  module DataModel
+    class Base
+      extend Forwardable
+      def_delegators :nodifier, :cdb
+      def_delegators :cdb, :lexicon, :source_table_exists?
+      def_delegators :lexicon,
+                     :concept_codes,
+                     :concept_ids,
+                     :concept_ids_by_name,
+                     :concepts,
+                     :concepts_table,
+                     :concepts_by_name,
+                     :concepts_ds,
+                     :concepts_to_codes,
+                     :descendants_of,
+                     :known_codes,
+                     :related_concept_ids
+
+      SCHEMAS = ConceptQL.schemas_dir.glob("*.yml").to_h do |schema_file|
+        [ schema_file.basename(".*").to_s.to_sym, Psych.load_file(schema_file) ]
+      end
+
+      attr_reader :operator, :nodifier
+
+      def initialize(operator, nodifier)
+        @operator = operator
+        @nodifier = nodifier
+      end
+
+      def rdbms
+        @rdbms ||= ConceptQL::Rdbms.generate(nodifier.database_type)
+      end
+
+      # The concept table that the display-name joins read. It is the
+      # Database's name_concepts_table when it has one, and the full concept
+      # table otherwise (bead conceptql-auu, seed conceptql-8u6).
+      def name_concepts_table(db)
+        table = cdb.opts[:name_concepts_table]
+        return concepts_table(db) unless table
+
+        db[table]
+      end
+
+      # The data columns whose concept ids the display-name joins look up, by
+      # table. A data model without those joins has none.
+      def name_concept_columns
+        {}
+      end
+
+      def query_modifier_for(column)
+        {
+          visit_source_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::PoSQueryModifier,
+          provider_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
+          context_specialty_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
+          drug_name: ConceptQL::QueryModifiers::Omopv4Plus::DrugQueryModifier,
+          admission_date: ConceptQL::QueryModifiers::Omopv4Plus::AdmissionDateQueryModifier,
+          provenance_type: ConceptQL::QueryModifiers::Omopv4Plus::ProvenanceQueryModifier
+        }[column]
+      end
+
+      def criterion_id
+        :criterion_id
+      end
+
+      def table_id(table = nil)
+        return :person_id if table.to_sym == :death
+
+        :criterion_id if table.nil?
+      end
+
+      def person_table
+        :person
+      end
+
+      def make_table_id(table)
+        "#{table}_id".to_sym
+      end
+
+      def make_fk_id(table)
+        make_table_id(table)
+      end
+
+      def person_id(table)
+        case table.to_sym
+        when :procedure_cost, :drug_cost
+          nil
+        else
+          :person_id
+        end
+      end
+
+      def schema
+        SCHEMAS[data_model]
+      end
+
+      def data_model
+        :omopv4_plus
+      end
+
+      def pk_by_domain(domain)
+        case domain.to_sym
+        when :death
+          :person_id
+        when :procedure_cost
+          :procedure_occurrence_id
+        when :drug_cost
+          :drug_exposure_id
+        else
+          "#{domain}_id".to_sym
+        end
+      end
+
+      def fk_by_domain(domain)
+        "#{domain}_id".to_sym
+      end
+
+      def query_columns
+        @query_columns ||= Hash[operator.scope.query_columns.zip(operator.scope.query_columns)]
+      end
+
+      def columns(opts = {})
+        table = get_table(opts)
+
+        cols_hash = if table.nil? && opts[:query_columns].nil?
+                      query_columns
+        else
+                      columns_in_table(table,
+                                       opts).merge(modifier_columns(table,
+                                                                    opts)).merge(nullified_columns(table, opts))
+        end.dup
+
+        except_keys = opts[:except] || []
+        if cols_hash.key?(:uuid)
+          if opts[:uuid]
+            except_keys << :uuid
+          elsif table
+            cols_hash.merge!(uuid: Sequel.cast(nil, String).as(:uuid))
+          end
+        end
+        col_keys = cols_hash.keys
+        col_keys -= except_keys
+        col_keys &= opts[:only] if opts[:only]
+
+        if (qualifier = opts[:qualifier])
+          cols_hash = cols_hash.transform_values { |c| Sequel.qualify(qualifier, c) }
+        end
+
+        cols_hash.merge!(replace(opts[:replace]))
+        cols_hash.values_at(*col_keys)
+      end
+
+      def modifier_columns(table, opts = {})
+        return {} if table.nil? && opts[:query_columns].nil?
+
+        remainder = query_columns.keys - (opts[:query_columns] ? opts[:query_columns].keys : columns_in_table(table).keys)
+        Hash[remainder.zip(remainder)]
+      end
+
+      def nullified_columns(table, opts = {})
+        return {} if table.nil? && opts[:query_columns].nil?
+
+        remainder = query_columns.keys
+        if opts[:query_columns]
+          remainder -= opts[:query_columns].keys
+        else
+          remainder -= table_cols(table)
+          remainder -= columns_in_table(table).keys
+          remainder -= applicable_query_modifiers(table).flat_map(&:provided_columns)
+        end
+
+        Hash[remainder.map { |r| [ r, rdbms.process(r, nil) ] }]
+      end
+
+      def selectify(query, opts = {})
+        ds = modify_query(query, get_table(opts)).select(*columns(opts))
+        ds = ds.from_self.select_append(rdbms.uuid.as(:uuid)) if opts[:uuid]
+        ds.from_self
+      end
+
+      def get_table(opts)
+        opts[:table] || table_by_domain(opts[:domain])
+      end
+
+      def modify_query(query, table)
+        return query if table.nil?
+
+        applicable_query_modifiers(table).each do |klass|
+          query = klass.new(query, operator, table, self).modified_query
+        end
+
+        query
+      end
+
+      def applicable_query_modifiers(table)
+        query_modifiers.values_at(*query_columns.keys).compact.select do |klass|
+          klass.has_required_columns?(table_cols(table))
+        end.uniq
+      end
+
+      def query_modifiers
+        {
+          visit_source_concept_id: query_modifier_for(:visit_source_concept_id),
+          provider_id: query_modifier_for(:provider_id),
+          context_specialty_concept_id: query_modifier_for(:context_specialty_concept_id),
+          drug_name: query_modifier_for(:drug_name),
+          file_provenance_type: query_modifier_for(:provenance_type),
+          admission_date: query_modifier_for(:admission_date),
+          value_as_number: query_modifier_for(:value_as_number)
+        }
+      end
+
+      def columns_in_table(table, opts = {})
+        return opts[:query_columns] unless opts[:query_columns].nil?
+
+        start_date, end_date = *date_columns(nil, table)
+        {
+          person_id: Sequel.expr(person_id(table)).as(:person_id),
+          criterion_id: Sequel.identifier(pk_by_domain(table)).as(:criterion_id),
+          criterion_table: Sequel.cast_string(table.to_s).as(:criterion_table),
+          criterion_domain: Sequel.cast_string((opts[:criterion_domain] || table_to_domain(table) || table).to_s).as(:criterion_domain),
+          start_date: start_date,
+          end_date: end_date,
+          source_value: Sequel.cast_string(source_value_column(table)).as(:source_value),
+          source_vocabulary_id: Sequel.cast_string(source_vocabulary_id(table)).as(:source_vocabulary_id)
+        }
+      end
+
+      def table_to_domain(table)
+        table
+      end
+
+      def replace(replace_hash)
+        return {} unless replace_hash
+
+        replace_hash.each_with_object({}) do |(column, value), h|
+          h[column] = rdbms.process(column, value)
+        end
+      end
+
+      def place_of_service_concept_id(query, domain)
+        place_of_service_concept_id_column(query, domain)
+      end
+
+      def determine_table(table_method)
+        return nil unless operator.respond_to?(table_method)
+
+        table = table_by_domain(operator.send(table_method))
+        table if schema.keys.include?(table)
+      end
+
+      def assign_column_to_table
+        schema.each_with_object({}) do |(table, table_info), cols|
+          columns_info = table_info[:columns]
+          column = yield table, columns_info.keys.map(&:to_s)
+          cols[table] = column ? column.to_sym : nil
+        end
+      end
+
+      def start_date_columns
+        @start_date_columns ||= assign_column_to_table do |_table, columns|
+          column = columns.select { |k| k =~ /start_date$/ }.first
+          column || columns.select { |k| k =~ /date$/ }.first
+        end
+      end
+
+      def end_date_columns
+        @end_date_columns ||= assign_column_to_table do |_table, columns|
+          column = columns.select { |k| k =~ /end_date$/ }.first
+          column || columns.select { |k| k =~ /date$/ }.first
+        end
+      end
+
+      def start_date_column(query, domain)
+        start_date_columns.merge(person: person_date_of_birth(query))[domain]
+      end
+
+      def end_date_column(query, domain)
+        end_date_columns.merge(person: person_date_of_birth(query))[domain]
+      end
+
+      def source_value_columns
+        @source_value_columns ||= assign_column_to_table do |table, columns|
+          reggy = /#{table.to_s.split('_').first}_source_value$/
+          column = columns.select { |k| k =~ reggy }.first
+          column || columns.select { |k| k =~ /source_value$/ }.first
+        end
+      end
+
+      def source_vocabulary_ids
+        @source_vocabulary_ids = assign_column_to_table do |table, columns|
+          next if %w[person death].any? { |tn| table.to_s =~ /#{tn}/ }
+
+          reggy = /#{table.to_s.split('_').first}_source_vocabulary_id/
+          column = columns.select { |k| k =~ reggy }.first
+          column || columns.select { |k| k =~ /_source_vocabulary_id/ }.first
+        end
+      end
+
+      def id_columns
+        @id_columns ||= assign_column_to_table do |table, columns|
+          reggy = /#{table}_id/
+          columns.select { |k| k =~ reggy }.first
+        end
+      end
+
+      def id_column(table)
+        id_columns[table]
+      end
+
+      def type_concept_id_columns
+        @type_concept_id_columns ||= assign_column_to_table do |_table, columns|
+          columns.select { |c| c =~ /_type_concept_id$/ }.first
+        end
+      end
+
+      def type_concept_id_column(domain)
+        type_concept_id_columns[table_by_domain(domain)]
+      end
+
+      def table_by_domain(domain)
+        domain
+      end
+
+      def period_table
+        :observation_period
+      end
+
+      def source_value_column(table)
+        source_value_columns[table]
+      end
+
+      def code_provenance_type_column(_query, domain)
+        {
+          condition_occurrence: :condition_type_concept_id,
+          death: :death_type_concept_id,
+          drug_exposure: :drug_type_concept_id,
+          observation: :observation_type_concept_id,
+          procedure_occurrence: :procedure_type_concept_id
+        }[domain]
+      end
+
+      def file_provenance_type_column(_query, domain)
+        {
+          condition_occurrence: :condition_type_concept_id,
+          death: :death_type_concept_id,
+          drug_exposure: :drug_type_concept_id,
+          observation: :observation_type_concept_id,
+          procedure_occurrence: :procedure_type_concept_id
+        }[domain]
+      end
+
+      def provider_id_column(_query, domain)
+        {
+          condition_occurrence: :associated_provider_id,
+          death: :death_type_concept_id,
+          drug_exposure: :prescribing_provider_id,
+          observation: :associated_provider_id,
+          person: :provider_id,
+          procedure_occurrence: :associated_provider_id,
+          provider: :provider_id
+        }[domain]
+      end
+
+      def place_of_service_concept_id_column(_query, domain)
+        return nil if domain.nil?
+        return Sequel.cast(:visit_source_concept_id, :Bigint) if table_cols(domain).include?(pos_table_fk)
+
+        nil
+      end
+
+      def pos_table_fk
+        :visit_occurrence_id
+      end
+
+      def person_date_of_birth(_query)
+        rdbms.cast_date(ConceptQL::Utils.assemble_date(:year_of_birth, :month_of_birth, :day_of_birth,
+                                                       database_type: nodifier.database_type))
+      end
+
+      def date_columns(query, table = nil)
+        sd = start_date_column(query, table)
+        sd = rdbms.cast_date(Sequel.expr(sd)).as(:start_date) unless sd == :start_date
+        ed = end_date_column(query, table)
+        unless ed == :end_date
+          ed = rdbms.cast_date(Sequel.function(:coalesce, Sequel.expr(ed),
+                                               start_date_column(query, table))).as(:end_date)
+        end
+        [ sd, ed ]
+      end
+
+      def table_to_sym(table)
+        case table
+        when Symbol
+          table = Sequel.split_symbol(table)[1].to_sym if Sequel.split_symbols?
+        when Sequel::SQL::AliasedExpression
+          table = table.expression
+        when Sequel::SQL::QualifiedIdentifier
+          table = table.column
+        when Sequel::SQL::Identifier
+          table = table.value
+        end
+        table.to_sym
+      end
+
+      def table_cols(table)
+        table = table_to_sym(table)
+        schema.fetch(table)[:columns].keys
+      end
+
+      def table_columns(*tables)
+        tables.map { |t| table_cols(t) }.flatten.tap do |o|
+          p o
+          p tables
+        end
+      end
+
+      def table_source_value(table)
+        source_value_columns.fetch(table_to_sym(table))
+      end
+
+      def source_vocabulary_id(table)
+        source_vocabulary_ids[table_to_sym(table)]
+      end
+
+      def table_is_missing?(db)
+        !(source_table_exists?(db, :observation_period) && source_table_exists?(db, :condition_occurrence))
+      end
+
+      def concept_id(table)
+        table_cols(table).find do |col|
+          col = col.to_s
+          next unless col =~ /concept_id/
+
+          table_parts = table.to_s.chomp("s").to_s.split("_")
+
+          possible_names = [ "concept_id" ]
+
+          until table_parts.empty?
+            possible_names.unshift([ table_parts.join("_"), possible_names.last ].join("_"))
+            table_parts.pop
+          end
+
+          possible_names.any? do |name|
+            col == name
+          end
+        end
+      end
+
+      def information_period_where_clause(arguments)
+        return if arguments.empty?
+
+        { plan_source_value: arguments }
+      end
+    end
+  end
+end

data/lib/conceptql/data_model/gdm.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module ConceptQL
+  module DataModel
+    class Gdm < Base
+      def query_modifier_for(column)
+        {
+          visit_source_concept_id: ConceptQL::QueryModifiers::Gdm::PoSQueryModifier,
+          provider_id: ConceptQL::QueryModifiers::Gdm::ProviderQueryModifier,
+          context_specialty_concept_id: ConceptQL::QueryModifiers::Gdm::ProviderQueryModifier,
+          drug_name: ConceptQL::QueryModifiers::Gdm::DrugQueryModifier,
+          admission_date: ConceptQL::QueryModifiers::Gdm::AdmissionDateQueryModifier,
+          provenance_type: ConceptQL::QueryModifiers::Gdm::ProvenanceQueryModifier,
+          value_as_number: ConceptQL::QueryModifiers::Gdm::LabQueryModifier
+        }[column]
+      end
+
+      def person_id(table = nil)
+        return :id if table == :patients
+
+        :patient_id
+      end
+
+      def person_table
+        :patients
+      end
+
+      def period_table
+        :information_periods
+      end
+
+      def criterion_id
+        Sequel.expr(:id).as(:criterion_id)
+      end
+
+      def table_id(table = nil)
+        return :criterion_id if table.nil?
+
+        Sequel.expr(make_table_id(table)).as(:criterion_id)
+      end
+
+      def make_table_id(_table)
+        :id
+      end
+
+      def fk_by_domain(domain)
+        table = table_by_domain(domain)
+        "#{table.to_s.gsub(/_id/, '').chomp('s')}_id".to_sym
+      end
+
+      def pk_by_domain(_domain)
+        :id
+      end
+
+      def base
+        :gdm
+      end
+
+      def name_concept_columns
+        {
+          clinical_codes: %i[clinical_code_concept_id],
+          drug_exposure_details: %i[dose_unit_concept_id],
+          measurement_details: %i[unit_concept_id result_as_concept_id],
+          admission_details: %i[admit_source_concept_id discharge_location_concept_id]
+        }
+      end
+
+      def table_by_domain(table)
+        return nil unless table
+
+        case table
+        when :person, :patients
+          :patients
+        when :death, :deaths
+          :deaths
+        when :observation_period, :information_periods
+          :information_periods
+        when :provider, :practitioners
+          :practitioners
+        when :collections
+          :collections
+        else
+          :clinical_codes
+        end
+      end
+
+      def table_to_domain(table)
+        {
+          patients: :person,
+          deaths: :death,
+          information_periods: :observation_period,
+          practitioners: :provider
+        }[table]
+      end
+
+      def data_model
+        :gdm
+      end
+
+      def start_date_column(_query, domain)
+        start_date_columns[domain]
+      end
+
+      def end_date_column(_query, domain)
+        end_date_columns[domain]
+      end
+
+      def id_column(_table)
+        :id
+      end
+
+      def pos_table_fk
+        :context_id
+      end
+
+      def source_vocabulary_ids
+        @source_vocabulary_ids = assign_column_to_table do |table, columns|
+          next if %w[patients death].any? { |tn| table.to_s =~ /#{tn}/ }
+
+          reggy = /#{table.to_s.split('_').first}_vocabulary_id/
+          column = columns.select { |k| k =~ reggy }.first
+          column || columns.select { |k| k =~ /_vocabulary_id/ }.first
+        end
+      end
+
+      def information_period_where_clause(arguments)
+        return if arguments.empty?
+
+        { information_type_concept_id: arguments.map(&:to_i) }
+      end
+
+      def code_provenance_type(_query, _domain)
+        :provenance_concept_id
+      end
+
+      def file_provenance_type(_query, _domain)
+        :source_type_concept_id
+      end
+
+      def file_provenance_types_vocab
+        %w[JIGSAW_FILE_PROVENANCE_TYPE JS_FILE_PROV_TYPE]
+      end
+
+      def code_provenance_types_vocab
+        %w[JIGSAW_CODE_PROVENANCE_TYPE JS_CODE_PROV_TYPE]
+      end
+
+      def table_is_missing?(db)
+        !(source_table_exists?(db, :patients) && source_table_exists?(db, :clinical_codes))
+      end
+    end
+  end
+end

data/lib/conceptql/data_model/gdm_wide.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "gdm"
+
+module ConceptQL
+  module DataModel
+    class GdmWide < Gdm
+      def query_modifier_for(column)
+        {
+          visit_source_concept_id: ConceptQL::QueryModifiers::GdmWide::PoSQueryModifier,
+          provider_id: ConceptQL::QueryModifiers::Gdm::ProviderQueryModifier,
+          context_specialty_concept_id: ConceptQL::QueryModifiers::Gdm::ProviderQueryModifier,
+          drug_name: ConceptQL::QueryModifiers::GdmWide::DrugQueryModifier,
+          admission_date: ConceptQL::QueryModifiers::GdmWide::AdmissionDateQueryModifier,
+          provenance_type: ConceptQL::QueryModifiers::GdmWide::ProvenanceQueryModifier,
+          value_as_number: ConceptQL::QueryModifiers::GdmWide::LabQueryModifier
+        }[column]
+      end
+
+      def table_by_domain(table)
+        tab = super(table)
+        tab = :observations if %i[clinical_codes collections].include?(tab)
+        tab
+      end
+
+      def data_model
+        :gdm_wide
+      end
+
+      def name_concept_columns
+        {
+          observations: %i[clinical_code_concept_id dose_unit_concept_id unit_concept_id result_as_concept_id
+                           admit_source_concept_id discharge_location_concept_id]
+        }
+      end
+
+      def table_is_missing?(db)
+        !source_table_exists?(db, :observations)
+      end
+    end
+  end
+end

data/lib/conceptql/data_model/omopv4_plus.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "base"
+module ConceptQL
+  module DataModel
+    class Omopv4Plus < Base
+      def base
+        :omop
+      end
+    end
+  end
+end

data/lib/conceptql/data_model.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "data_model/gdm"
+require_relative "data_model/gdm_wide"
+require_relative "data_model/omopv4_plus"
+require_relative "rdbms"
+
+module ConceptQL
+  module DataModel
+    def self.get(data_model, opts = {})
+      operator = opts[:operator]
+      nodifier = opts[:nodifier]
+
+      case data_model
+      when :gdm
+        Gdm.new(operator, nodifier)
+      when :gdm_wide
+        GdmWide.new(operator, nodifier)
+      when :omopv4_plus
+        Omopv4Plus.new(operator, nodifier)
+      else
+        raise "No DataModel defined for #{data_model.inspect}"
+      end
+    end
+
+    def self.for(operator, nodifier)
+      get(nodifier.data_model, nodifier: nodifier, operator: operator)
+    end
+  end
+end