conceptql 0.3.0 → 0.4.0

data/lib/conceptql/lexicon/lexicon_ohdsi.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "lexicon_strategy"
+
+module ConceptQL
+  class LexiconOhdsi < LexiconStrategy
+    class << self
+      def db_has_all_vocabulary_tables?(db)
+        %i[concept_ancestor concept concept_relationship vocabulary].all? { |t| db.table_exists?(t) }
+      end
+    end
+
+    def vocabularies_query
+      db[:vocabulary]
+        .select(Sequel[:vocabulary_id].as(:id),
+                Sequel[:vocabulary_id].as(:omopv5_vocabulary_id),
+                Sequel[:vocabulary_name].as(:vocabulary_short_name),
+                Sequel[:vocabulary_name].as(:vocabulary_full_name),
+                Sequel[nil].as(:domain),
+                Sequel.expr(1).as(:from_lexicon))
+        .from_self
+    end
+
+    def vocabularies
+      vocabularies_query.all
+    end
+
+    def is_a_relationships(_data_db)
+      db[:concept_relationship]
+        .select(
+          Sequel[:concept_id_1].as(:concept_1_id),
+          Sequel[:concept_id_2].as(:concept_2_id),
+          :relationship_id
+        )
+        .where { Sequel.function(:lower, :relationship_id) =~ "is a" }
+        .from_self
+    end
+
+    def ancestors_table(_data_db)
+      db[:concept_ancestor].select(
+        Sequel[:ancestor_concept_id].as(:ancestor_id),
+        Sequel[:descendant_concept_id].as(:descendant_id)
+      ).from_self
+    end
+
+    def concepts_table(_data_db, _some_schema = nil)
+      db[:concept].select(
+        Sequel[:concept_id].as(:id),
+        :concept_code,
+        :vocabulary_id,
+        Sequel[:concept_name].as(:concept_text)
+      ).from_self
+    end
+
+    def table_is_missing?(_data_db)
+      !db.table_exists?(:concept)
+    end
+
+    def strategy
+      :ohdsi
+    end
+  end
+end

data/lib/conceptql/lexicon/lexicon_strategy.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  class LexiconStrategy
+    attr_reader :db
+
+    def initialize(db)
+      @db = db
+    end
+
+    # A lookup is always scoped to a vocabulary. `vocabulary_id` is polymorphic:
+    # a String, an Integer (ReadOmop reports 17), or an Array of vocabulary names
+    # (the provenance-type pairs), which Sequel turns into an IN clause.
+    #
+    # There is deliberately no wildcard. `'*'` is meaningful elsewhere in the
+    # operator family, but only in `arguments` — Vocabulary#select_all? reads it
+    # there and *skips* the lexicon call entirely. An earlier version of this
+    # method honoured `'*'` here too, which no caller ever used and which read as
+    # support for cross-vocabulary lookup that does not exist.
+    def concepts(data_db, vocabulary_id, codes = [])
+      if vocabulary_id.blank? || Array(vocabulary_id).include?("*")
+        raise ArgumentError,
+              "concepts requires a specific vocabulary, got #{vocabulary_id.inspect}. " \
+              "There is no wildcard lookup; pass a vocabulary name, an Array of them, " \
+              "or an id. ('*' belongs in arguments, where Vocabulary#select_all? handles it.)"
+      end
+
+      ds = concepts_table(data_db)
+
+      ds = ds.where(vocabulary_id: vocabulary_id)
+      ds = ds.where(Sequel.function(:lower, :concept_code) => Array(codes).map(&:downcase)) unless codes.blank?
+
+      ds
+    end
+
+    def concepts_by_name(data_db, names = [])
+      ds = concepts_table(data_db)
+
+      ds.where(Sequel.function(:lower, :concept_text) => Array(names).map(&:downcase))
+    end
+
+    def descendants_of(_data_db, concept_ids_or_ds)
+      where_values = Array(concept_ids_or_ds).flatten.dup
+
+      descendants = ancestors_table(db)
+                    .where(ancestor_id: where_values)
+                    .select(:descendant_id)
+
+      unless where_values.empty?
+        union_clause = db.values(where_values.map { |v| [ v ] })
+        descendants = descendants.union(union_clause).distinct
+      end
+
+      descendants.select_map(:descendant_id)
+    end
+
+    def concept_ids(_data_db, vocabulary_id, codes = [])
+      concepts(db, vocabulary_id, codes)
+        .select_map(:id)
+    end
+
+    # Every concept code in the vocabulary (or vocabularies).
+    def concept_codes(data_db, vocabulary_id)
+      concepts(data_db, vocabulary_id).select_map(:concept_code)
+    end
+
+    def concept_ids_by_name(data_db, names)
+      concepts_by_name(data_db, names).select_map(:id)
+    end
+
+    # The mappings table will tell us what other concepts have been directly
+    # mapped to the concepts passed in
+    def related_concept_ids(data_db, *ids)
+      ids = ids.flatten
+      other_ids = is_a_relationships(data_db)
+                  .where(concept_2_id: ids)
+                  .select_map(:concept_1_id)
+      other_ids + ids
+    end
+
+    # A caller can answer "is the vocabulary empty?" with a block. The
+    # Lexicon does, so that the answer comes from its cache.
+    def known_codes(data_db, vocabulary_id, codes)
+      return codes if db_is_mock?(data_db)
+      return codes if block_given? ? yield : vocabulary_is_empty?(data_db, vocabulary_id)
+
+      concepts_ds(data_db, vocabulary_id, codes).select_map(:concept_code)
+    rescue Sequel::DatabaseError
+      []
+    end
+
+    # A caller can answer "is the concept table missing?" with a block. The
+    # Lexicon does, so that the answer comes from its cache.
+    def concepts_to_codes(data_db, vocabulary_id, codes = [])
+      missing = db.nil? || (block_given? ? yield : table_is_missing?(data_db))
+      return codes.map { |code| [ code, nil ] } if missing
+
+      concepts(db, vocabulary_id, codes).select_map(%i[concept_code concept_text])
+    end
+
+    def vocabulary_is_empty?(data_db, vocabulary_id)
+      concepts_table(data_db).where(vocabulary_id: vocabulary_id).count.zero?
+    end
+
+    def concepts_ds(data_db, vocabulary_id, codes)
+      concepts_table(data_db)
+        .where(vocabulary_id: vocabulary_id, concept_code: codes)
+        .select(Sequel[:concept_code].as(:concept_code), Sequel[:concept_text].as(:concept_text))
+        .from_self
+    end
+
+    def db_is_mock?(_data_db)
+      db.is_a?(Sequel::Mock::Database)
+    end
+  end
+end

data/lib/conceptql/lexicon.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require_relative "lexicon/lexicon_gdm"
+require_relative "lexicon/lexicon_no_db"
+require_relative "lexicon/lexicon_ohdsi"
+
+module ConceptQL
+  class Lexicon
+    extend Forwardable
+
+    # Lookups that return plain values. With cache: true each result is kept
+    # for the life of this Lexicon, which is the life of its Database, so the
+    # cache never outlives one export (seed conceptql-a5f). Methods that return
+    # datasets are not cached: their query runs later, in the caller.
+    CACHED_LOOKUPS = %i[
+      concept_codes
+      concept_ids
+      concept_ids_by_name
+      concepts_to_codes
+      descendants_of
+      known_codes
+      related_concept_ids
+      table_is_missing?
+      vocabulary_is_empty?
+    ].freeze
+
+    def_delegators :@strategy,
+                   :strategy,
+                   :concepts,
+                   :concepts_table,
+                   :concepts_by_name,
+                   :concepts_ds,
+                   :vocabularies,
+                   :vocabularies_query
+
+    # The cached results keyed on [method, arguments], or nil when caching is
+    # off. Its size is the number of distinct lookups.
+    attr_reader :cache
+
+    def initialize(lexicon_db, dataset_db = nil, strategy: nil, cache: false)
+      @strategy = strategy || determine_strategy(lexicon_db, dataset_db)
+      @cache = {} if cache
+    end
+
+    (CACHED_LOOKUPS - %i[concepts_to_codes known_codes table_is_missing?]).each do |name|
+      define_method(name) do |*args|
+        lookup(name, args) { @strategy.public_send(name, *args) }
+      end
+    end
+
+    # The strategy's known_codes first asks whether the vocabulary is empty.
+    # It asks through the block, so that question also reaches the cache.
+    def known_codes(data_db, vocabulary_id, codes)
+      lookup(:known_codes, [ data_db, vocabulary_id, codes ]) do
+        @strategy.known_codes(data_db, vocabulary_id, codes) { vocabulary_is_empty?(data_db, vocabulary_id) }
+      end
+    end
+
+    # The strategy's concepts_to_codes first asks whether the concept table is
+    # missing. It asks through the block, so that question also reaches the cache.
+    def concepts_to_codes(data_db, vocabulary_id, codes = [])
+      lookup(:concepts_to_codes, [ data_db, vocabulary_id, codes ]) do
+        @strategy.concepts_to_codes(data_db, vocabulary_id, codes) { table_is_missing?(data_db) }
+      end
+    end
+
+    # Only "the table is there" is kept. A missing table (or a database error,
+    # which Sequel's table_exists? turns into "missing") is checked again on
+    # the next call, so one transient error does not last the whole export.
+    def table_is_missing?(data_db)
+      lookup(:table_is_missing?, [ data_db ], keep: ->(missing) { !missing }) do
+        @strategy.table_is_missing?(data_db)
+      end
+    end
+
+    # Runs the lookup, or with caching on returns the result kept for the same
+    # method and arguments. A result that keep rejects is returned, not kept.
+    def lookup(name, args, keep: nil)
+      return yield unless cache
+
+      # Copy Array arguments so a caller that mutates one later cannot
+      # change a key. Results are frozen so a mutation fails loudly.
+      key = [ name, args.map { |arg| arg.is_a?(Array) ? arg.dup.freeze : arg } ]
+      cache.fetch(key) do
+        result = yield.dup.freeze
+        cache[key] = result if keep.nil? || keep.call(result)
+        result
+      end
+    end
+    private :lookup
+
+    def determine_strategy(lexicon_db, dataset_db)
+      [ dataset_db, lexicon_db ].compact.each do |db|
+        lexicon_classes.each do |klass|
+          next unless klass.db_has_all_vocabulary_tables?(db)
+
+          return (klass.new(db))
+        end
+      end
+      LexiconNoDB.new(Sequel.mock(host: :postgres))
+    end
+
+    def lexicon_classes
+      classes = []
+      classes << LexiconOhdsi unless ENV["LEXICON_GDM_ONLY"]
+      classes << LexiconGDM unless ENV["LEXICON_OHDSI_ONLY"]
+      classes
+    end
+  end
+end

data/lib/conceptql/nodifier.rb

@@ -1,45 +1,75 @@
-require_relative 'operators/operator'
+# frozen_string_literal: true
+
+require_relative "operators/operator"
 
 module ConceptQL
   class Nodifier
-    attr :scope, :data_model, :database_type, :algorithm_fetcher
+    attr_reader :cdb, :scope, :data_model, :database_type, :algorithm_fetcher
 
-    def initialize(opts={})
-      @scope = opts[:scope] || Scope.new(opts)
-      @data_model = opts[:data_model] || :omopv4
-      @database_type = opts[:database_type] || :impala
-      @algorithm_fetcher = opts[:algorithm_fetcher] || (proc do |alg|
+    def initialize(cdb, opts = {})
+      @cdb = cdb
+      @scope = opts[:scope] || Scope.new(opts.delete(:scope_opts) || {})
+      @data_model = get_data_model(opts)
+      @database_type = opts[:database_type] || cdb.database_type
+      @algorithm_fetcher = opts[:algorithm_fetcher] || (proc do |_alg|
         nil
       end)
     end
 
+    def get_data_model(opts)
+      (opts[:data_model] || cdb.opts[:data_model] || ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym
+    end
+
     def create(operator, *values)
-      if operator.to_s == 'algorithm'
-        statement, desc = algorithm_fetcher.call(values.first)
+      operator = operator.to_s.downcase
+      if operator == "algorithm"
+        statement, = algorithm_fetcher.call(values.first)
         if statement
           create(*statement)
         else
           invalid_op(operator, values, "invalid algorithm", values.first)
         end
-      elsif klass = operators[operator.to_s]
-        klass.new(self, *values)
+      elsif (klass = fetch_op(operator))
+        klass.new(self, operator, *values)
       else
         invalid_op(operator, values, "invalid operator", operator)
       end
     end
 
     def to_metadata(opts = {})
-      Hash[operators.map { |k, v| [k, v.to_metadata(k, opts)]}.select { |k, v| !v[:categories].empty? }.sort_by { |k, v| v[:name] }]
+      Hash[operators.map do |k, v|
+        [ k, v.to_metadata(k, opts) ]
+      end.reject { |_k, v| v[:categories].empty? }.sort_by { |_k, v| v[:name] }]
+    end
+
+    # The operators this nodifier can create: its Database's registry for
+    # its data model.
+    def operators
+      @operators ||= cdb.operators(@data_model)
     end
 
     private
 
-    def operators
-      @operators ||= Operators.operators[@data_model]
+    def fetch_op(operator)
+      operators[alias_for(operator)]
+    end
+
+    def alias_for(operator)
+      operator_aliases[operator] || operator
+    end
+
+    def operator_aliases
+      @operator_aliases ||= operators.flat_map do |id, klass|
+        next unless klass.aliases.present?
+
+        klass.aliases.map { |klass_alias| [ klass_alias, id ] }
+      end.compact.to_h
     end
 
     def invalid_op(operator, values, *error_args)
-      options = values.extract_options!.merge(errors: [error_args])
+      options = values.pop if values.last.is_a?(Hash)
+      options ||= {}
+      options = options.merge(errors: [ error_args ])
       values << options
       Operators::Invalid.new(self, operator, *values)
     end

data/lib/conceptql/operators/after.rb

@@ -1,42 +1,45 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class After < TemporalOperator
       register __FILE__
 
-      desc <<-EOF
-Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand results (RHR).
-Any result in the LHR with a start_date that occurs after the earliest end_date in the RHR is passed through.
-All other results are discarded, including all results in the RHR.
-L-------N-------L
-R-----R
-   R-----R
-        L-----Y----L
-      EOF
+      desc "For each person, passes along left hand records with a start_date that occurs after the earliest end_date of a right hand record."
 
+      allows_at_least_option
       within_skip :after
 
-      def right_stream(db)
-        unless compare_all?
-          right.evaluate(db).from_self.group_by(:person_id).select(:person_id, Sequel.function(:min, :end_date).as(:end_date)).as(:r)
+      def right_stream_query(db)
+        if compare_all?
+          right.evaluate(db).from_self
         else
-          right.evaluate(db).from_self.as(:r)
+          right.evaluate(db).from_self.select_group(*matching_columns).select_append(Sequel.function(:min,
+                                                                                                     :end_date).as(:end_date))
         end
       end
 
-      def occurrences_column
-        :end_date
-      end
-
       def where_clause
-        Proc.new { l__start_date > r__end_date }
+        after_date = r_end_date
+
+        after_date = adjust_date(at_least_option, after_date) if at_least_option
+
+        clause = Sequel.expr(l_start_date > after_date)
+
+        clause &= l_start_date <= within_end if within_option
+
+        clause
       end
 
       def compare_all?
-        !(options.keys & [:within, :at_least, :occurrences]).empty?
+        !(options.keys & [ :within ]).empty?
+      end
+
+      def rhs_function
+        compare_all? ? nil : :min
       end
     end
   end
 end
-

data/lib/conceptql/operators/any_overlap.rb

@@ -1,17 +1,17 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class AnyOverlap < TemporalOperator
       register __FILE__
 
-      desc 'If a result in the LHR overlaps in any way a result in the RHR, it is passed through.'
+      desc "For each person, passes along left hand records with a date range that overlaps in any way with a right hand record's date_range."
+
       def where_clause
-        l_partly_in_r = Sequel.expr { r__start_date <= l__start_date }.&(Sequel.expr { l__start_date <= r__end_date })
-        r_partly_in_l = Sequel.expr { l__start_date <= r__start_date }.&(Sequel.expr { r__start_date <= l__end_date })
-        l_partly_in_r.|(r_partly_in_l)
+        ((within_start <= l_start_date) & (l_start_date <= within_end)) | ((l_start_date <= within_start) & (within_start <= l_end_date))
       end
     end
   end
 end
-

data/lib/conceptql/operators/before.rb

@@ -1,36 +1,44 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class Before < TemporalOperator
       register __FILE__
 
-      desc <<-EOF
-Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand results (RHR).
-Any result in the LHR with an end_date that occurs before the most recent start_date of the RHR is passed through.
-All other results are discarded, including all results in the RHR.
-      EOF
+      desc "For each person, passes along left hand records with an end_date that occurs before the most recent start_date of a right hand record."
 
+      allows_at_least_option
       within_skip :before
 
-      def right_stream(db)
-        unless compare_all?
-          right.evaluate(db).from_self.group_by(:person_id).select(:person_id, Sequel.function(:max, :start_date).as(:start_date)).as(:r)
+      def right_stream_query(db)
+        if compare_all?
+          right.evaluate(db).from_self
         else
-          right.evaluate(db).from_self.as(:r)
+          right.evaluate(db).from_self.select_group(*matching_columns).select_append(Sequel.function(:max,
+                                                                                                     :start_date).as(:start_date))
         end
       end
 
-      def within_column
-        :l__end_date
-      end
-
       def where_clause
-        Proc.new { l__end_date < r__start_date }
+        before_date = r_start_date
+
+        before_date = adjust_date(at_least_option, before_date, true) if at_least_option
+
+        before_clause = Sequel.expr(l_end_date < before_date)
+
+        before_clause &= l_end_date >= within_start if within_option
+
+        before_clause
       end
 
       def compare_all?
-        !(options.keys & [:within, :at_least, :occurrences]).empty?
+        !(options.keys & [ :within ]).empty?
+      end
+
+      def rhs_function
+        compare_all? ? nil : :min
       end
     end
   end

data/lib/conceptql/operators/binary_operator_operator.rb

@@ -1,5 +1,6 @@
-require_relative 'operator'
-require 'facets/string/titlecase'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
@@ -14,36 +15,74 @@ module ConceptQL
       basic_type :filter
 
       def upstreams
-        [left]
+        [ left ]
       end
 
-      def display_name
-        self.class.name.split('::').last.snakecase.titlecase
+      def code_list(db)
+        left.code_list(db) + right.code_list(db)
       end
 
-      def code_list(db)
-        puts "BinaryOperatorOperator.code_list #{self.inspect}"
-        code_lists = [left, right].map do | upstream_op |
-          upstream_op.code_list(db)
-        end
-        code_lists.flatten(1)
+      attr_reader :left, :right
+
+      def events_per_patient
+        left.events_per_patient
+      end
+
+      def multiple_vocabularies
+        left.multiple_vocabularies
       end
 
-      attr :left, :right
+      def accept(visitor, opts = {})
+        visitor.visit(self)
+        left.accept(visitor, opts)
+        right.accept(visitor, opts) unless opts[:avoid_rhs]
+      end
+
+      def all_upstreams
+        [ left, right ]
+      end
 
       private
 
+      def pp_upstreams
+        { left: left, right: right }
+      end
+
+      def join_columns(opts = {})
+        join_columns_option.map { |c| Sequel.expr([ [ Sequel[:l][c], Sequel[opts[:qualifier] || :r][c] ] ]) }
+      end
+
+      def join_columns_option
+        (column_names_option(:join_columns) || []) + matching_columns
+      end
+
       def annotate_values(db, opts = {})
         h = {}
         h[:left] = left.annotate(db, opts) if left
         h[:right] = right.annotate(db, opts) if right
-        [options.merge(h), *arguments]
+        [ options.merge(h), *arguments ]
       end
 
       def create_upstreams
         @left = to_op(options[:left]) if options[:left].is_a?(Array)
         @right = to_op(options[:right])  if options[:right].is_a?(Array)
       end
+
+      def left_stream(db)
+        left_stream_query(db)
+      end
+
+      def left_stream_query(db)
+        left.evaluate(db).from_self(alias: :l)
+      end
+
+      def right_stream(db)
+        right_stream_query(db).as(:r)
+      end
+
+      def right_stream_query(db)
+        right.evaluate(db).from_self(alias: :r)
+      end
     end
   end
 end