conceptql 0.4.1 → 0.6.0

data/lib/conceptql/behaviors/metadatable.rb

@@ -38,6 +38,15 @@ module ConceptQL
       @conceptql_spec_id = value
     end
 
+    # Marks an operator as deprecated. It still runs. Operator#validate adds
+    # the message to its annotation as a warning, and its metadata says
+    # deprecated: true. A subclass does not inherit the mark.
+    def deprecated(message = nil)
+      return @deprecated unless message
+
+      @deprecated = message
+    end
+
     def predominant_domains(*values)
       return @predominant_domains if values.empty?
 
@@ -150,11 +159,12 @@ module ConceptQL
         max_upstreams: @max_upstreams || 0,
         arguments: @arguments || [],
         options: process_options(@options),
-        predominant_domains: @domains || @predominant_domains || [],
+        # Strings, as JSON gives them: domains (for example death) are symbols
+        predominant_domains: (@domains || @predominant_domains || []).map(&:to_s),
         desc: get_desc,
         categories: @categories || [],
         basic_type: @basic_type,
-        deprecated: nil, # kept for metadata consumers; no operator is deprecated
+        deprecated: (true if deprecated), # nil for an operator that is not deprecated
         arity: arity,
         aliases: @aliases,
         spec_id: spec_id
@@ -176,8 +186,10 @@ module ConceptQL
       end
     end
 
+    # The desc that the operator set (no_desc sets ""), else the standard
+    # vocabulary text. A subclass does not inherit its parent's desc.
     def get_desc
-      @get_desc ||= standard_description
+      @desc || standard_description
     end
 
     def process_options(options)

data/lib/conceptql/behaviors/provenanceable.rb

@@ -1,26 +1,100 @@
 # frozen_string_literal: true
 
 module ConceptQL
+  # Provenance keywords select records by their file provenance
+  # (file_provenance_type) and their code provenance (code_provenance_type).
+  #
+  # A plain keyword, such as "inpatient" or "primary", names a concept in the
+  # file or the code provenance vocabulary. It keeps a record when either
+  # column is in the descendants of that concept.
+  #
+  # A keyword "<file keyword>:<code keyword>", such as
+  # "inpatient:primary_or_first", keeps a record when its file provenance is
+  # in the descendants of the file keyword AND its code provenance is in the
+  # descendants of the code keyword.
+  #
+  # Several keywords are ORed: a record is kept when any keyword keeps it.
+  #
+  # The keyword "*" keeps a record when either column is any provenance
+  # concept. Because keywords are ORed, "*" with other keywords is "*".
   module Provenanceable
+    PAIR_SEPARATOR = ":"
+    ANY = "*"
+
     def self.included(base)
       base.require_column :file_provenance_type
       base.require_column :code_provenance_type
     end
 
-    def prov_of(db, ancestors)
-      dm.descendants_of(db,
-                        dm.concept_ids(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten,
-                                       ancestors))
+    # No keyword at all is a validation error of Provenance. It still builds
+    # the condition of "*", as an invalid pair still builds a condition.
+    def build_where_from_codes(db, codes)
+      return plain_condition(db, []) if codes.empty? || codes.include?(ANY)
+
+      pairs, plain = codes.partition { |code| pair_keyword?(code) }
+      conditions = pairs.map { |pair| pair_condition(db, pair) }
+      conditions.unshift(plain_condition(db, plain)) if plain.any?
+      return conditions.first if conditions.one?
+
+      Sequel.|(*conditions)
     end
 
-    def build_where_from_codes(db, codes)
+    # The keywords as written that name no provenance concept. A pair is
+    # recognized only when it has one separator, a file keyword on the left
+    # and a code keyword on the right. Case is compared exactly, as the lookup
+    # compares it: the provenance vocabularies have no vocabulary_formats row
+    # that declares case-insensitivity (seed conceptql-oji ruling "b"). So
+    # "Inpatient" names no concept, and the concept codes are in lower case.
+    # "*" is known. It is not a half of a pair.
+    def find_bad_keywords(db, codes)
+      pairs, plain = (codes - [ ANY ]).partition { |code| pair_keyword?(code) }
+      known = known_keywords(db, all_provenance_types_vocab)
+      bad = plain.reject { |code| known.include?(code.to_s) }
+      bad += pairs.reject { |pair| known_pair?(db, pair) }
+      codes & bad
+    end
+
+    private
+
+    def pair_keyword?(code)
+      code.to_s.include?(PAIR_SEPARATOR)
+    end
+
+    def known_pair?(db, pair)
+      file, code, *rest = pair.split(PAIR_SEPARATOR, -1)
+      rest.empty? &&
+        known_keywords(db, dm.file_provenance_types_vocab).include?(file) &&
+        known_keywords(db, dm.code_provenance_types_vocab).include?(code)
+    end
+
+    # The concept codes of the vocabularies.
+    def known_keywords(db, vocabularies)
+      dm.concept_codes(db, vocabularies)
+    end
+
+    def all_provenance_types_vocab
+      [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten
+    end
+
+    def prov_of(db, vocabularies, ancestors)
+      dm.descendants_of(db, dm.concept_ids(db, vocabularies, ancestors))
+    end
+
+    def plain_condition(db, codes)
       Sequel.|(
-        { file_provenance_type: prov_of(db, codes) }, { code_provenance_type: prov_of(db, codes) }
+        { file_provenance_type: prov_of(db, all_provenance_types_vocab, codes) },
+        { code_provenance_type: prov_of(db, all_provenance_types_vocab, codes) }
       )
     end
 
-    def find_bad_keywords(db, codes)
-      codes - dm.concept_codes(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten)
+    # An invalid pair (validation reports it) still builds a condition. A
+    # half with no concept matches nothing, so the pair keeps no record.
+    def pair_condition(db, pair)
+      file, code = pair.split(PAIR_SEPARATOR, 2)
+      Sequel.&(
+        { file_provenance_type: prov_of(db, dm.file_provenance_types_vocab, [ file ]) },
+        { code_provenance_type: prov_of(db, dm.code_provenance_types_vocab, [ code ]) }
+      )
     end
   end
 end

data/lib/conceptql/behaviors/utilizable.rb

@@ -41,13 +41,17 @@ module ConceptQL
       # With the Database's share_utilization option, the list is read from a
       # table that every statement on the Database shares. The WITH entry only
       # carries the list to Scope#with_ctes, which takes it out of the SQL and
-      # creates the table when the query first reads rows.
+      # creates the table when the query first reads rows. With
+      # CONCEPTQL_AVOID_CTES, with_ctes extracts nothing, so the scope keeps
+      # the table instead.
       def evaluate(db)
         cdb = nodifier.cdb
         return super unless cdb.opts[:share_utilization]
 
         list = select_it(query(db))
         name = cdb.shared_utilization_table(list, scope)
+        return scope.add_shared_table(db, name, list) if ConceptQL.avoid_ctes?
+
         db[name].with(name, list, Scope::SHARED_TABLE => true)
       end
 
@@ -170,8 +174,6 @@ module ConceptQL
       end
 
       def override_columns
-        return unless gdm?
-
         {
           length_of_stay: (rdbms.days_between(Sequel[:admission_date],
                                               discharge_date_column) + 1).as(:length_of_stay),

data/lib/conceptql/cli.rb

@@ -94,7 +94,7 @@ module ConceptQL
       given by the connection options (or sequelizer's configuration).
     DESC
     option :counts, type: :boolean, default: false, desc: "annotate with per-domain row/person counts (needs a DB)"
-    option :data_model, desc: "data model: gdm, gdm_wide, omopv4_plus (default: $CONCEPTQL_DATA_MODEL or gdm)"
+    option :data_model, desc: "data model: gdm, gdm_wide (default: $CONCEPTQL_DATA_MODEL or gdm)"
     def render_json(statement_file)
       cdb = if options[:counts]
               ConceptQL::Database.new(db(options.except("counts", "data_model")),

data/lib/conceptql/data_model/base.rb

@@ -10,6 +10,7 @@ module ConceptQL
       def_delegators :nodifier, :cdb
       def_delegators :cdb, :lexicon, :source_table_exists?
       def_delegators :lexicon,
+                     :code_meanings,
                      :concept_codes,
                      :concept_ids,
                      :concept_ids_by_name,
@@ -18,6 +19,7 @@ module ConceptQL
                      :concepts_by_name,
                      :concepts_ds,
                      :concepts_to_codes,
+                     :concepts_to_codes_in_class,
                      :descendants_of,
                      :known_codes,
                      :related_concept_ids
@@ -47,79 +49,14 @@ module ConceptQL
         db[table]
       end
 
-      # The data columns whose concept ids the display-name joins look up, by
-      # table. A data model without those joins has none.
-      def name_concept_columns
-        {}
-      end
-
-      def query_modifier_for(column)
-        {
-          visit_source_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::PoSQueryModifier,
-          provider_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
-          context_specialty_concept_id: ConceptQL::QueryModifiers::Omopv4Plus::ProviderQueryModifier,
-          drug_name: ConceptQL::QueryModifiers::Omopv4Plus::DrugQueryModifier,
-          admission_date: ConceptQL::QueryModifiers::Omopv4Plus::AdmissionDateQueryModifier,
-          provenance_type: ConceptQL::QueryModifiers::Omopv4Plus::ProvenanceQueryModifier
-        }[column]
-      end
-
-      def criterion_id
-        :criterion_id
-      end
-
-      def table_id(table = nil)
-        return :person_id if table.to_sym == :death
-
-        :criterion_id if table.nil?
-      end
-
-      def person_table
-        :person
-      end
-
-      def make_table_id(table)
-        "#{table}_id".to_sym
-      end
-
       def make_fk_id(table)
         make_table_id(table)
       end
 
-      def person_id(table)
-        case table.to_sym
-        when :procedure_cost, :drug_cost
-          nil
-        else
-          :person_id
-        end
-      end
-
       def schema
         SCHEMAS[data_model]
       end
 
-      def data_model
-        :omopv4_plus
-      end
-
-      def pk_by_domain(domain)
-        case domain.to_sym
-        when :death
-          :person_id
-        when :procedure_cost
-          :procedure_occurrence_id
-        when :drug_cost
-          :drug_exposure_id
-        else
-          "#{domain}_id".to_sym
-        end
-      end
-
-      def fk_by_domain(domain)
-        "#{domain}_id".to_sym
-      end
-
       def query_columns
         @query_columns ||= Hash[operator.scope.query_columns.zip(operator.scope.query_columns)]
       end
@@ -231,10 +168,6 @@ module ConceptQL
         }
       end
 
-      def table_to_domain(table)
-        table
-      end
-
       def replace(replace_hash)
         return {} unless replace_hash
 
@@ -276,14 +209,6 @@ module ConceptQL
         end
       end
 
-      def start_date_column(query, domain)
-        start_date_columns.merge(person: person_date_of_birth(query))[domain]
-      end
-
-      def end_date_column(query, domain)
-        end_date_columns.merge(person: person_date_of_birth(query))[domain]
-      end
-
       def source_value_columns
         @source_value_columns ||= assign_column_to_table do |table, columns|
           reggy = /#{table.to_s.split('_').first}_source_value$/
@@ -292,81 +217,10 @@ module ConceptQL
         end
       end
 
-      def source_vocabulary_ids
-        @source_vocabulary_ids = assign_column_to_table do |table, columns|
-          next if %w[person death].any? { |tn| table.to_s =~ /#{tn}/ }
-
-          reggy = /#{table.to_s.split('_').first}_source_vocabulary_id/
-          column = columns.select { |k| k =~ reggy }.first
-          column || columns.select { |k| k =~ /_source_vocabulary_id/ }.first
-        end
-      end
-
-      def id_columns
-        @id_columns ||= assign_column_to_table do |table, columns|
-          reggy = /#{table}_id/
-          columns.select { |k| k =~ reggy }.first
-        end
-      end
-
-      def id_column(table)
-        id_columns[table]
-      end
-
-      def type_concept_id_columns
-        @type_concept_id_columns ||= assign_column_to_table do |_table, columns|
-          columns.select { |c| c =~ /_type_concept_id$/ }.first
-        end
-      end
-
-      def type_concept_id_column(domain)
-        type_concept_id_columns[table_by_domain(domain)]
-      end
-
-      def table_by_domain(domain)
-        domain
-      end
-
-      def period_table
-        :observation_period
-      end
-
       def source_value_column(table)
         source_value_columns[table]
       end
 
-      def code_provenance_type_column(_query, domain)
-        {
-          condition_occurrence: :condition_type_concept_id,
-          death: :death_type_concept_id,
-          drug_exposure: :drug_type_concept_id,
-          observation: :observation_type_concept_id,
-          procedure_occurrence: :procedure_type_concept_id
-        }[domain]
-      end
-
-      def file_provenance_type_column(_query, domain)
-        {
-          condition_occurrence: :condition_type_concept_id,
-          death: :death_type_concept_id,
-          drug_exposure: :drug_type_concept_id,
-          observation: :observation_type_concept_id,
-          procedure_occurrence: :procedure_type_concept_id
-        }[domain]
-      end
-
-      def provider_id_column(_query, domain)
-        {
-          condition_occurrence: :associated_provider_id,
-          death: :death_type_concept_id,
-          drug_exposure: :prescribing_provider_id,
-          observation: :associated_provider_id,
-          person: :provider_id,
-          procedure_occurrence: :associated_provider_id,
-          provider: :provider_id
-        }[domain]
-      end
-
       def place_of_service_concept_id_column(_query, domain)
         return nil if domain.nil?
         return Sequel.cast(:visit_source_concept_id, :Bigint) if table_cols(domain).include?(pos_table_fk)
@@ -374,15 +228,6 @@ module ConceptQL
         nil
       end
 
-      def pos_table_fk
-        :visit_occurrence_id
-      end
-
-      def person_date_of_birth(_query)
-        rdbms.cast_date(ConceptQL::Utils.assemble_date(:year_of_birth, :month_of_birth, :day_of_birth,
-                                                       database_type: nodifier.database_type))
-      end
-
       def date_columns(query, table = nil)
         sd = start_date_column(query, table)
         sd = rdbms.cast_date(Sequel.expr(sd)).as(:start_date) unless sd == :start_date
@@ -414,10 +259,7 @@ module ConceptQL
       end
 
       def table_columns(*tables)
-        tables.map { |t| table_cols(t) }.flatten.tap do |o|
-          p o
-          p tables
-        end
+        tables.flat_map { |t| table_cols(t) }
       end
 
       def table_source_value(table)
@@ -428,10 +270,6 @@ module ConceptQL
         source_vocabulary_ids[table_to_sym(table)]
       end
 
-      def table_is_missing?(db)
-        !(source_table_exists?(db, :observation_period) && source_table_exists?(db, :condition_occurrence))
-      end
-
       def concept_id(table)
         table_cols(table).find do |col|
           col = col.to_s
@@ -451,12 +289,6 @@ module ConceptQL
           end
         end
       end
-
-      def information_period_where_clause(arguments)
-        return if arguments.empty?
-
-        { plan_source_value: arguments }
-      end
     end
   end
 end

data/lib/conceptql/data_model/gdm.rb

@@ -115,8 +115,10 @@ module ConceptQL
         :context_id
       end
 
+      # The schema of a data model does not change, so the answer is kept, as
+      # for start_date_columns and the other column lookups.
       def source_vocabulary_ids
-        @source_vocabulary_ids = assign_column_to_table do |table, columns|
+        @source_vocabulary_ids ||= assign_column_to_table do |table, columns|
           next if %w[patients death].any? { |tn| table.to_s =~ /#{tn}/ }
 
           reggy = /#{table.to_s.split('_').first}_vocabulary_id/

data/lib/conceptql/data_model.rb

@@ -2,11 +2,29 @@
 
 require_relative "data_model/gdm"
 require_relative "data_model/gdm_wide"
-require_relative "data_model/omopv4_plus"
 require_relative "rdbms"
 
 module ConceptQL
   module DataModel
+    SUPPORTED = %i[gdm gdm_wide].freeze
+
+    # Raised when a caller asks for a data model that ConceptQL does not have.
+    class UnsupportedError < ArgumentError
+      REMOVED = { omopv4_plus: "0.5.0" }.freeze
+
+      def initialize(data_model)
+        message = "ConceptQL has no data model #{data_model.inspect}. Use one of: #{SUPPORTED.join(', ')}."
+        if (version = REMOVED[data_model])
+          message = "The #{data_model} data model was removed in ConceptQL #{version}. Use one of: #{SUPPORTED.join(', ')}."
+        end
+        super(message)
+      end
+    end
+
+    def self.assert_supported!(data_model)
+      raise UnsupportedError, data_model unless SUPPORTED.include?(data_model)
+    end
+
     def self.get(data_model, opts = {})
       operator = opts[:operator]
       nodifier = opts[:nodifier]
@@ -16,10 +34,8 @@ module ConceptQL
         Gdm.new(operator, nodifier)
       when :gdm_wide
         GdmWide.new(operator, nodifier)
-      when :omopv4_plus
-        Omopv4Plus.new(operator, nodifier)
       else
-        raise "No DataModel defined for #{data_model.inspect}"
+        raise UnsupportedError, data_model
       end
     end
 

data/lib/conceptql/database.rb

@@ -28,6 +28,7 @@ module ConceptQL
       @opts = ConceptQL::Utils.rekey(opts, rekey_values: true)
 
       @opts[:data_model] ||= (ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym
+      ConceptQL::DataModel.assert_supported!(@opts[:data_model])
 
       self.class.db_extensions(db, @opts[:data_model], @opts[:use_cold_col])
       db_type = db ? db.database_type.to_sym : :postgres
@@ -94,8 +95,10 @@ module ConceptQL
       end
 
       # The LEXICON_URL database, or nil when it is unset. Lexicon tries the
-      # data db first and, when neither has the vocabulary tables, falls back
-      # to LexiconNoDB, so no database (and no sqlite3 gem) is needed here.
+      # data db first. When neither has the vocabulary tables, a real data db
+      # raises Lexicon::NoVocabularyError, and a db-less Database (db nil or a
+      # Sequel mock) falls back to LexiconNoDB, so no database (and no sqlite3
+      # gem) is needed here.
       def make_lexicon_db
         return unless ENV["LEXICON_URL"]
 
@@ -125,6 +128,9 @@ module ConceptQL
       @source_tables.fetch(key) { db.table_exists?(table) && (@source_tables[key] = true) }
     end
 
+    # The Lexicon passed as lexicon:, or one built from the data db and the
+    # LEXICON_URL database. Database.new calls it, so a real data db without
+    # vocabulary tables raises Lexicon::NoVocabularyError there.
     def lexicon
       @lexicon ||= Lexicon.new(self.class.lexicon_db, db, cache: @opts[:cache_lookups])
     end
@@ -139,6 +145,7 @@ module ConceptQL
     # differs from this Database's.
     def operators(data_model = opts[:data_model])
       data_model = data_model.to_sym
+      ConceptQL::DataModel.assert_supported!(data_model)
       @operators ||= {}
       @operators[data_model] ||= begin
         vocabulary_operators = Vocabularies::DynamicVocabularies.new(lexicon).operators(data_model)

data/lib/conceptql/date_adjuster.rb

@@ -26,8 +26,7 @@ module ConceptQL
 
       origin_column = column
 
-      chrs = str.chars
-      while (chr = chrs.shift) =~ /[res]/i
+      prefix.each_char do |chr|
         if chr.downcase == "r"
           reverse = true
         else
@@ -53,13 +52,20 @@ module ConceptQL
       op.rdbms.cast_date(adjusted_date)
     end
 
-    # Returns an array of strings that represent date modifiers
+    # Returns an array of [units, quantity] pairs that represent date modifiers.
+    # An "r" in the prefix reverses them, as it does in #adjust.
     def adjustments(reverse = false)
-      @adjustments ||= parse(str, reverse)
+      reverse ||= prefix.match?(/r/i)
+      (@adjustments ||= {})[reverse] ||= parse(str.delete_prefix(prefix), reverse)
     end
 
     private
 
+    # The leading e/r/s characters, which pick the origin column and the direction
+    def prefix
+      str[/\A[res]*/i]
+    end
+
     def lookup
       {
         "y" => :years,
@@ -71,7 +77,8 @@ module ConceptQL
 
     def parse(str, reverse)
       return [] if str.nil? || str.empty?
-      return [ [ lookup["d"], str.to_i ] ] if str.match(/^[-+]?\d+$/)
+      # A bare number means days
+      return [ [ lookup["d"], reverse ? -str.to_i : str.to_i ] ] if str.match?(/\A[-+]?\d+\z/)
 
       str.downcase.scan(/([-+]?\d*[dwmy])/).map do |adjustment|
         adjustment = adjustment.first

data/lib/conceptql/lexicon/lexicon_gdm.rb

@@ -4,9 +4,12 @@ require_relative "lexicon_strategy"
 
 module ConceptQL
   class LexiconGDM < LexiconStrategy
+    # The GDM vocabulary tables. A db that has all four can be the lexicon.
+    VOCABULARY_TABLES = %i[ancestors concepts mappings vocabularies].freeze
+
     class << self
       def db_has_all_vocabulary_tables?(db)
-        %i[ancestors concepts mappings vocabularies].all? { |t| db.table_exists?(t) }
+        VOCABULARY_TABLES.all? { |t| db.table_exists?(t) }
       end
     end
 

data/lib/conceptql/lexicon/lexicon_ohdsi.rb

@@ -4,9 +4,12 @@ require_relative "lexicon_strategy"
 
 module ConceptQL
   class LexiconOhdsi < LexiconStrategy
+    # The OHDSI vocabulary tables. A db that has all four can be the lexicon.
+    VOCABULARY_TABLES = %i[concept concept_ancestor concept_relationship vocabulary].freeze
+
     class << self
       def db_has_all_vocabulary_tables?(db)
-        %i[concept_ancestor concept concept_relationship vocabulary].all? { |t| db.table_exists?(t) }
+        VOCABULARY_TABLES.all? { |t| db.table_exists?(t) }
       end
     end
 
@@ -56,6 +59,21 @@ module ConceptQL
       !db.table_exists?(:concept)
     end
 
+    # A caller can answer "is the concept table missing?" with a block. A
+    # concept table without concept_class_id answers nil, as a lexicon
+    # without classes does.
+    def concepts_to_codes_in_class(data_db, vocabulary_id, concept_class_id, codes)
+      return nil if codes.blank? || db_is_mock?(data_db)
+      return nil if block_given? ? yield : table_is_missing?(data_db)
+
+      db[:concept]
+        .where(vocabulary_id: vocabulary_id, concept_class_id: concept_class_id)
+        .where(code_condition(vocabulary_id, codes))
+        .select_map(%i[concept_code concept_name])
+    rescue Sequel::DatabaseError
+      nil
+    end
+
     def strategy
       :ohdsi
     end