conceptql 0.4.1 → 0.5.0

data/lib/conceptql/behaviors/utilizable.rb

@@ -170,8 +170,6 @@ module ConceptQL
       end
 
       def override_columns
-        return unless gdm?
-
         {
           length_of_stay: (rdbms.days_between(Sequel[:admission_date],
                                               discharge_date_column) + 1).as(:length_of_stay),

data/lib/conceptql/cli.rb

@@ -94,7 +94,7 @@ module ConceptQL
       given by the connection options (or sequelizer's configuration).
     DESC
     option :counts, type: :boolean, default: false, desc: "annotate with per-domain row/person counts (needs a DB)"
-    option :data_model, desc: "data model: gdm, gdm_wide, omopv4_plus (default: $CONCEPTQL_DATA_MODEL or gdm)"
+    option :data_model, desc: "data model: gdm, gdm_wide (default: $CONCEPTQL_DATA_MODEL or gdm)"
     def render_json(statement_file)
       cdb = if options[:counts]
               ConceptQL::Database.new(db(options.except("counts", "data_model")),

data/lib/conceptql/data_model/base.rb

@@ -10,6 +10,7 @@ module ConceptQL
       def_delegators :nodifier, :cdb
       def_delegators :cdb, :lexicon, :source_table_exists?
       def_delegators :lexicon,
+                     :code_meanings,
                      :concept_codes,
                      :concept_ids,
                      :concept_ids_by_name,
@@ -47,79 +48,14 @@ module ConceptQL
         db[table]
       end
 
-      # The data columns whose concept ids the display-name joins look up, by
-      # table. A data model without those joins has none.
-      def name_concept_columns
-        {}
-      end
-
-      def query_modifier_for(column)
-        {
-          visit_source_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::PoSQueryModifier,
-          provider_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
-          context_specialty_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
-          drug_name: ConceptQL::QueryModifiers::Omopv4Plus::DrugQueryModifier,
-          admission_date: ConceptQL::QueryModifiers::Omopv4Plus::AdmissionDateQueryModifier,
-          provenance_type: ConceptQL::QueryModifiers::Omopv4Plus::ProvenanceQueryModifier
-        }[column]
-      end
-
-      def criterion_id
-        :criterion_id
-      end
-
-      def table_id(table = nil)
-        return :person_id if table.to_sym == :death
-
-        :criterion_id if table.nil?
-      end
-
-      def person_table
-        :person
-      end
-
-      def make_table_id(table)
-        "#{table}_id".to_sym
-      end
-
       def make_fk_id(table)
         make_table_id(table)
       end
 
-      def person_id(table)
-        case table.to_sym
-        when :procedure_cost, :drug_cost
-          nil
-        else
-          :person_id
-        end
-      end
-
       def schema
         SCHEMAS[data_model]
       end
 
-      def data_model
-        :omopv4_plus
-      end
-
-      def pk_by_domain(domain)
-        case domain.to_sym
-        when :death
-          :person_id
-        when :procedure_cost
-          :procedure_occurrence_id
-        when :drug_cost
-          :drug_exposure_id
-        else
-          "#{domain}_id".to_sym
-        end
-      end
-
-      def fk_by_domain(domain)
-        "#{domain}_id".to_sym
-      end
-
       def query_columns
         @query_columns ||= Hash[operator.scope.query_columns.zip(operator.scope.query_columns)]
       end
@@ -231,10 +167,6 @@ module ConceptQL
         }
       end
 
-      def table_to_domain(table)
-        table
-      end
-
       def replace(replace_hash)
         return {} unless replace_hash
 
@@ -276,14 +208,6 @@ module ConceptQL
         end
       end
 
-      def start_date_column(query, domain)
-        start_date_columns.merge(person: person_date_of_birth(query))[domain]
-      end
-
-      def end_date_column(query, domain)
-        end_date_columns.merge(person: person_date_of_birth(query))[domain]
-      end
-
       def source_value_columns
         @source_value_columns ||= assign_column_to_table do |table, columns|
           reggy = /#{table.to_s.split('_').first}_source_value$/
@@ -292,81 +216,10 @@ module ConceptQL
         end
       end
 
-      def source_vocabulary_ids
-        @source_vocabulary_ids = assign_column_to_table do |table, columns|
-          next if %w[person death].any? { |tn| table.to_s =~ /#{tn}/ }
-
-          reggy = /#{table.to_s.split('_').first}_source_vocabulary_id/
-          column = columns.select { |k| k =~ reggy }.first
-          column || columns.select { |k| k =~ /_source_vocabulary_id/ }.first
-        end
-      end
-
-      def id_columns
-        @id_columns ||= assign_column_to_table do |table, columns|
-          reggy = /#{table}_id/
-          columns.select { |k| k =~ reggy }.first
-        end
-      end
-
-      def id_column(table)
-        id_columns[table]
-      end
-
-      def type_concept_id_columns
-        @type_concept_id_columns ||= assign_column_to_table do |_table, columns|
-          columns.select { |c| c =~ /_type_concept_id$/ }.first
-        end
-      end
-
-      def type_concept_id_column(domain)
-        type_concept_id_columns[table_by_domain(domain)]
-      end
-
-      def table_by_domain(domain)
-        domain
-      end
-
-      def period_table
-        :observation_period
-      end
-
       def source_value_column(table)
         source_value_columns[table]
       end
 
-      def code_provenance_type_column(_query, domain)
-        {
-          condition_occurrence: :condition_type_concept_id,
-          death: :death_type_concept_id,
-          drug_exposure: :drug_type_concept_id,
-          observation: :observation_type_concept_id,
-          procedure_occurrence: :procedure_type_concept_id
-        }[domain]
-      end
-
-      def file_provenance_type_column(_query, domain)
-        {
-          condition_occurrence: :condition_type_concept_id,
-          death: :death_type_concept_id,
-          drug_exposure: :drug_type_concept_id,
-          observation: :observation_type_concept_id,
-          procedure_occurrence: :procedure_type_concept_id
-        }[domain]
-      end
-
-      def provider_id_column(_query, domain)
-        {
-          condition_occurrence: :associated_provider_id,
-          death: :death_type_concept_id,
-          drug_exposure: :prescribing_provider_id,
-          observation: :associated_provider_id,
-          person: :provider_id,
-          procedure_occurrence: :associated_provider_id,
-          provider: :provider_id
-        }[domain]
-      end
-
       def place_of_service_concept_id_column(_query, domain)
         return nil if domain.nil?
         return Sequel.cast(:visit_source_concept_id, :Bigint) if table_cols(domain).include?(pos_table_fk)
@@ -374,15 +227,6 @@ module ConceptQL
         nil
       end
 
-      def pos_table_fk
-        :visit_occurrence_id
-      end
-
-      def person_date_of_birth(_query)
-        rdbms.cast_date(ConceptQL::Utils.assemble_date(:year_of_birth, :month_of_birth, :day_of_birth,
-                                                       database_type: nodifier.database_type))
-      end
-
       def date_columns(query, table = nil)
         sd = start_date_column(query, table)
         sd = rdbms.cast_date(Sequel.expr(sd)).as(:start_date) unless sd == :start_date
@@ -428,10 +272,6 @@ module ConceptQL
         source_vocabulary_ids[table_to_sym(table)]
       end
 
-      def table_is_missing?(db)
-        !(source_table_exists?(db, :observation_period) && source_table_exists?(db, :condition_occurrence))
-      end
-
       def concept_id(table)
         table_cols(table).find do |col|
           col = col.to_s
@@ -451,12 +291,6 @@ module ConceptQL
           end
         end
       end
-
-      def information_period_where_clause(arguments)
-        return if arguments.empty?
-
-        { plan_source_value: arguments }
-      end
     end
   end
 end

data/lib/conceptql/data_model.rb

@@ -2,11 +2,29 @@
 
 require_relative "data_model/gdm"
 require_relative "data_model/gdm_wide"
-require_relative "data_model/omopv4_plus"
 require_relative "rdbms"
 
 module ConceptQL
   module DataModel
+    SUPPORTED = %i[gdm gdm_wide].freeze
+
+    # Raised when a caller asks for a data model that ConceptQL does not have.
+    class UnsupportedError < ArgumentError
+      REMOVED = { omopv4_plus: "0.5.0" }.freeze
+
+      def initialize(data_model)
+        message = "ConceptQL has no data model #{data_model.inspect}. Use one of: #{SUPPORTED.join(', ')}."
+        if (version = REMOVED[data_model])
+          message = "The #{data_model} data model was removed in ConceptQL #{version}. Use one of: #{SUPPORTED.join(', ')}."
+        end
+        super(message)
+      end
+    end
+
+    def self.assert_supported!(data_model)
+      raise UnsupportedError, data_model unless SUPPORTED.include?(data_model)
+    end
+
     def self.get(data_model, opts = {})
       operator = opts[:operator]
       nodifier = opts[:nodifier]
@@ -16,10 +34,8 @@ module ConceptQL
         Gdm.new(operator, nodifier)
       when :gdm_wide
         GdmWide.new(operator, nodifier)
-      when :omopv4_plus
-        Omopv4Plus.new(operator, nodifier)
       else
-        raise "No DataModel defined for #{data_model.inspect}"
+        raise UnsupportedError, data_model
       end
     end
 

data/lib/conceptql/database.rb

@@ -28,6 +28,7 @@ module ConceptQL
       @opts = ConceptQL::Utils.rekey(opts, rekey_values: true)
 
       @opts[:data_model] ||= (ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym
+      ConceptQL::DataModel.assert_supported!(@opts[:data_model])
 
       self.class.db_extensions(db, @opts[:data_model], @opts[:use_cold_col])
       db_type = db ? db.database_type.to_sym : :postgres
@@ -139,6 +140,7 @@ module ConceptQL
     # differs from this Database's.
     def operators(data_model = opts[:data_model])
       data_model = data_model.to_sym
+      ConceptQL::DataModel.assert_supported!(data_model)
       @operators ||= {}
       @operators[data_model] ||= begin
         vocabulary_operators = Vocabularies::DynamicVocabularies.new(lexicon).operators(data_model)

data/lib/conceptql/lexicon/lexicon_strategy.rb

@@ -8,9 +8,9 @@ module ConceptQL
       @db = db
     end
 
-    # A lookup is always scoped to a vocabulary. `vocabulary_id` is polymorphic:
-    # a String, an Integer (ReadOmop reports 17), or an Array of vocabulary names
-    # (the provenance-type pairs), which Sequel turns into an IN clause.
+    # A lookup is always scoped to a vocabulary. `vocabulary_id` is a String or
+    # an Array of vocabulary names (the provenance-type pairs), which Sequel turns
+    # into an IN clause.
     #
     # There is deliberately no wildcard. `'*'` is meaningful elsewhere in the
     # operator family, but only in `arguments` — Vocabulary#select_all? reads it
@@ -98,6 +98,26 @@ module ConceptQL
       concepts(db, vocabulary_id, codes).select_map(%i[concept_code concept_text])
     end
 
+    # The distinct names of each code that has more than one name, as
+    # { downcased code => [name, ...] } with the names sorted. A vocabulary
+    # can reuse a code for a new meaning: DRG 410 meant chemotherapy until
+    # 2007-09-30 and a biliary tract procedure after that. OHDSI keeps each
+    # meaning as its own concept row, and the GDM vocabulary keeps both names.
+    # Names that differ only in case or spacing count as one name.
+    # A caller can answer "is the concept table missing?" with a block.
+    def code_meanings(data_db, vocabulary_id, codes)
+      return {} if codes.blank? || db.nil? || db_is_mock?(data_db)
+      return {} if block_given? ? yield : table_is_missing?(data_db)
+
+      concepts(db, vocabulary_id, codes)
+        .select_map(%i[concept_code concept_text])
+        .group_by { |code, _| code.downcase }
+        .transform_values { |rows| rows.map(&:last).compact.uniq { |name| name.downcase.split.join(" ") }.sort }
+        .select { |_, names| names.length > 1 }
+    rescue Sequel::DatabaseError
+      {}
+    end
+
     def vocabulary_is_empty?(data_db, vocabulary_id)
       concepts_table(data_db).where(vocabulary_id: vocabulary_id).count.zero?
     end

data/lib/conceptql/lexicon.rb

@@ -14,6 +14,7 @@ module ConceptQL
     # cache never outlives one export (seed conceptql-a5f). Methods that return
     # datasets are not cached: their query runs later, in the caller.
     CACHED_LOOKUPS = %i[
+      code_meanings
       concept_codes
       concept_ids
       concept_ids_by_name
@@ -43,7 +44,7 @@ module ConceptQL
       @cache = {} if cache
     end
 
-    (CACHED_LOOKUPS - %i[concepts_to_codes known_codes table_is_missing?]).each do |name|
+    (CACHED_LOOKUPS - %i[code_meanings concepts_to_codes known_codes table_is_missing?]).each do |name|
       define_method(name) do |*args|
         lookup(name, args) { @strategy.public_send(name, *args) }
       end
@@ -65,6 +66,13 @@ module ConceptQL
       end
     end
 
+    # Like concepts_to_codes, it asks about the concept table through the block.
+    def code_meanings(data_db, vocabulary_id, codes)
+      lookup(:code_meanings, [ data_db, vocabulary_id, codes ]) do
+        @strategy.code_meanings(data_db, vocabulary_id, codes) { table_is_missing?(data_db) }
+      end
+    end
+
     # Only "the table is there" is kept. A missing table (or a database error,
     # which Sequel's table_exists? turns into "missing") is checked again on
     # the next call, so one transient error does not last the whole export.

data/lib/conceptql/operators/co_reported.rb

@@ -17,7 +17,10 @@ module ConceptQL
         vocabularies.length > 1
       end
 
-      desc "Passes along all events that were co-reported in the same record in the source data."
+      desc "Passes along the events of its upstream streams that were reported in the same source record " \
+           "(the same context). A record is kept only if every upstream stream " \
+           "has at least one event in it. If any one upstream has no event in a record, the events of all " \
+           "streams in that record are dropped, without a warning."
 
       allows_many_upstreams
       category "Combine Streams"
@@ -27,14 +30,6 @@ module ConceptQL
       require_column :context_id
 
       def query(db)
-        if gdm?
-          gdm(db)
-        else
-          events_with_common_visits(db).inject { |q, query| q.union(query, all: true) }
-        end
-      end
-
-      def gdm(db)
         contexteds = upstreams.map do |stream|
           [ stream.cte_name(:contextified), contextify(db, stream) ]
         end.to_h
@@ -84,28 +79,6 @@ module ConceptQL
         # .select_append(Sequel[:c][:context_id].as(:context_id))
         # .from_self
       end
-
-      private
-
-      def visit_occurrence_ids_in_common(db)
-        @visit_occurrence_ids_in_common ||= upstream_queries(db).map do |q|
-          q.select(:visit_occurrence_id)
-        end.inject do |q, query|
-          q.from_self(alias: :tab1)
-           .join(query.as(:tab2), visit_occurrence_id: :visit_occurrence_id)
-           .select(Sequel[:tab1][:visit_occurrence_id].as(:visit_occurrence_id))
-        end
-      end
-
-      def events_with_common_visits(db)
-        upstream_queries(db).map { |q| q.where(visit_occurrence_id: visit_occurrence_ids_in_common(db)).from_self }
-      end
-
-      def upstream_queries(db)
-        @upstream_queries ||= upstreams.map do |expression|
-          expression.evaluate(db).select(*query_cols).from_self
-        end
-      end
     end
   end
 end

data/lib/conceptql/operators/hospice.rb

@@ -5,8 +5,7 @@ require_relative "operator"
 module ConceptQL
   module Operators
     class Hospice < Operator
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
       include ConceptQL::Behaviors::Utilizable
 
       desc "Selects admission records of type 'hospice'"

data/lib/conceptql/operators/hospitalization.rb

@@ -5,8 +5,7 @@ require_relative "operator"
 module ConceptQL
   module Operators
     class Hospitalization < Operator
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
       include ConceptQL::Behaviors::Utilizable
 
       desc "Selects admission records of type 'inpatient'"

data/lib/conceptql/operators/information_periods.rb

@@ -23,9 +23,7 @@ module ConceptQL
       end
 
       def source_table
-        tab = dm.table_by_domain(domain)
-        tab = :payer_plan_period if tab == :observation_period && arguments.present?
-        tab
+        dm.table_by_domain(domain)
       end
 
       def where_clause

data/lib/conceptql/operators/one_in_two_out.rb

@@ -10,8 +10,7 @@ module ConceptQL
     class OneInTwoOut < Operator
       include ConceptQL::Behaviors::SingleEvent
 
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
 
       desc "Identifies an event that appears either once in an inpatient setting or twice within a specified interval in an outpatient setting."
 

data/lib/conceptql/operators/operator.rb

@@ -9,7 +9,7 @@ module ConceptQL
     # Built-in operators only: the classes that call Operator.register when
     # their file loads. Vocabulary operators are not in here; each
     # ConceptQL::Database builds its own from its lexicon (Database#operators).
-    STATIC_OPERATORS = { omopv4_plus: {}, gdm: {}, gdm_wide: {} }.freeze
+    STATIC_OPERATORS = { gdm: {}, gdm_wide: {} }.freeze
 
     SELECTED_COLUMNS = %i[person_id
                           criterion_id
@@ -348,14 +348,6 @@ module ConceptQL
         end
       end
 
-      def omopv4_plus?
-        data_model == :omopv4_plus
-      end
-
-      def gdm?
-        %i[gdm gdm_wide].include?(data_model)
-      end
-
       def wide?
         %i[gdm_wide].include?(data_model)
       end
@@ -467,10 +459,10 @@ module ConceptQL
         cast_column(:file_provenance_type, dm.provenance_type_column(query, table))
       end
 
-      def provider_id(query, table)
+      def provider_id(query, _table)
         return :provider_id if query_columns(query).include?(:provider_id)
 
-        cast_column(:provider_id, dm.provider_id_column(query, table))
+        cast_column(:provider_id)
       end
 
       def determine_tables

data/lib/conceptql/operators/provenance.rb

@@ -16,8 +16,7 @@ module ConceptQL
     #
     # Multiple provenances can be specified at once
     class Provenance < Operator
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
 
       desc "Passes along records with the indicated provenance (e.g. inpatient, outpatient, file type)."
 

data/lib/conceptql/operators/race.rb

@@ -38,11 +38,7 @@ module ConceptQL
       end
 
       def source_table
-        if gdm?
-          :patients
-        else
-          :person
-        end
+        :patients
       end
 
       def query(db)
@@ -52,7 +48,7 @@ module ConceptQL
 
         c_ids = (concept_ids + actual_ids).map { |i| [ i, race_descendents[i] ] }.flatten.compact.uniq.sort
 
-        c_ids = dm.related_concept_ids(db, c_ids) if gdm?
+        c_ids = dm.related_concept_ids(db, c_ids)
 
         q = db.from(source_table)
         if words.any? { |w| w.match(/unknown/i) }

data/lib/conceptql/operators/read.rb

@@ -18,25 +18,15 @@ module ConceptQL
       conceptql_spec_id "vocabulary"
 
       def query(db)
-        gdm? ? gdm(db) : omopv4(db)
-      end
-
-      def gdm(db)
         ops(db).first.evaluate(db)
       end
 
-      def omopv4(db)
-        streams = ops(db).map { |op| op.evaluate(db) }
-
-        streams.inject { |q, query| q.union(query, all: true) }.from_self
-      end
-
       def domains(db)
         ops.first.domains(db)
       end
 
       def query_cols
-        gdm? ? ops.first.query_cols : ops.map(&:query_cols).flatten.uniq
+        ops.first.query_cols
       end
 
       def required_columns
@@ -53,40 +43,11 @@ module ConceptQL
         ops.flat_map { |op| op.describe_codes(db, codes) }
       end
 
-      def klasses
-        @klasses ||= {
-          condition_occurrence: ReadCondition,
-          procedure_occurrence: ReadProcedure,
-          drug_exposure: ReadDrug,
-          observation: ReadObservation
-        }
-      end
-
       class ReadBase < ConceptQL::Operators::Vocabulary
         preferred_name "READ"
         argument :read_codes, type: :codelist, vocab: "Read"
       end
 
-      class ReadOmopBase < ReadBase
-        include ConceptQL::Vocabularies::Behaviors::Omopish
-
-        def vocabulary_id
-          17
-        end
-
-        def source_table
-          table
-        end
-
-        def domain
-          table
-        end
-
-        def domains(_db)
-          Array(domain)
-        end
-      end
-
       class ReadGDM < ReadBase
         include ConceptQL::Vocabularies::Behaviors::Gdmish
         include ConceptQL::Behaviors::Labish
@@ -103,66 +64,6 @@ module ConceptQL
           [ :condition_occurrence ]
         end
       end
-
-      class ReadCondition < ReadOmopBase
-        def table
-          :procedure_occurrence
-        end
-
-        def source_column
-          :procedure_source_value
-        end
-
-        def concept_column
-          :procedure_concept_id
-        end
-      end
-
-      class ReadProcedure < ReadOmopBase
-        def table
-          :procedure_occurrence
-        end
-
-        def source_column
-          :procedure_source_value
-        end
-
-        def concept_column
-          :procedure_concept_id
-        end
-      end
-
-      class ReadObservation < ReadOmopBase
-        include ConceptQL::Behaviors::Labish
-
-        def table
-          :observation
-        end
-
-        def source_column
-          :observation_source_value
-        end
-
-        def concept_column
-          :observation_concept_id
-        end
-      end
-
-      class ReadDrug < ReadOmopBase
-        include ConceptQL::Behaviors::Drugish
-
-        def table
-          :drug_exposure
-        end
-
-        def source_column
-          :drug_source_value
-        end
-
-        def concept_column
-          :drug_concept_id
-        end
-      end
     end
   end
 end