conceptql 0.2.0 → 0.4.0

data/lib/conceptql/utils.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/array/conversions"
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/string/inflections"
+require "json"
+require "open3"
+require "timeout"
+
+module ConceptQL
+  module Utils
+    class << self
+      def rekey(h, opts = {})
+        # Thanks to: https://stackoverflow.com/a/10721936
+        case h
+        when Hash
+          Hash[
+            h.map do |k, v|
+              [ k.respond_to?(:to_sym) ? k.to_sym : k, rekey(v, opts) ]
+            end
+          ]
+        when Sequel::Dataset
+          h
+        when Enumerable
+          h.map { |v| rekey(v, opts) }
+        else
+          return h unless opts[:rekey_values]
+          return h.to_sym if h.respond_to?(:to_sym)
+
+          h
+        end
+      end
+
+      def timed_capture(*commands)
+        # Heavily adapted from:
+        # https://gist.github.com/lpar/1032297#gistcomment-1738285
+        opts = extract_opts!(commands)
+        timeout = opts.fetch(:timeout)
+
+        stdin, stdout, stderr, wait_thread = Open3.popen3(*commands)
+        wait_thread[:timed_out] = false
+        stdin.puts opts[:stdin_data] if opts[:stdin_data]
+        stdin.close
+
+        # Purposefully NOT using Timeout.rb because in general it is a dangerous API!
+        # http://blog.headius.com/2008/02/rubys-threadraise-threadkill-timeoutrb.html
+        Thread.new do
+          sleep timeout
+          if wait_thread.alive?
+            begin
+              wait_thread[:timed_out] = true
+              # please note: we are assuming the command will create ONE process (not handling subprocs / proc groups)
+              command = "kill -9 #{wait_thread.pid}"
+              system(command)
+            rescue(Errno::ESRCH)
+              # Do nothing!
+            end
+          end
+        end
+
+        if (es = wait_thread.value.exitstatus) && !es.zero?
+          raise stderr.read
+        end
+
+        out = stdout.read
+        stdout.close
+
+        if wait_thread[:timed_out]
+          raise Timeout::Error, "Command #{commands} failed to complete after #{timeout} seconds"
+        end
+
+        out
+      end
+
+      def extract_opts!(arr)
+        return {} unless arr.last.is_a?(Hash)
+
+        arr.pop
+      end
+
+      def assemble_date(*symbols)
+        opts = extract_opts!(symbols)
+        tab = opts[:table] ? Sequel[opts[:table].to_sym] : Sequel
+
+        strings = symbols.map do |symbol|
+          sub = "2000"
+          col = Sequel.cast_string(tab[symbol])
+          if symbol != :year_of_birth
+            sub = "01"
+            col = Sequel.function(:lpad, col, 2, "0")
+          end
+          Sequel.function(:coalesce, col, Sequel.expr(sub))
+        end
+
+        strings_with_dashes = strings.zip([ "-" ] * (symbols.length - 1)).flatten.compact
+        Sequel.join(strings_with_dashes)
+      end
+
+      def schema_dump(db)
+        schema = db.tables.sort.map do |t|
+          columns = db.schema(t).map do |column_name, column_info|
+            info = column_info.slice(:type, :allow_null, :primary_key, :default, :comment)
+            info[:null] = info.delete(:allow_null) ? nil : false
+            info[:primary_key] = info.delete(:primary_key) ? true : nil
+            [ column_name, info.compact ]
+          end.to_h
+          [ t, { columns: columns } ]
+        end.to_h
+        schema.to_yaml
+      end
+    end
+  end
+end

data/lib/conceptql/utils_cli.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "thor"
+require "sequelizer"
+require "psych"
+require_relative "utils"
+
+module ConceptQL
+  class UtilsCLI < Thor
+    include Sequelizer
+
+    desc "dump_schema", "Dumps out the schema for all tables in YAML format"
+    def dump_schema
+      puts ConceptQL::Utils.schema_dump(db)
+    end
+  end
+end

data/lib/conceptql/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ConceptQL
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/conceptql/visitors/metadata_extractor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Visitors
+    class MetadataExtractor
+      attr_reader :operators
+
+      def initialize
+        @operators = []
+      end
+
+      def visit(operator)
+        operators << operator
+      end
+
+      def results
+        {
+          unique_operators: operator_names.uniq.sort,
+          operators_in_preorder: operator_names,
+          codeset_only: codeset_only?,
+          num_selection_nodes: num_selection_nodes,
+          max_depth: max_depth
+        }
+      end
+
+      def operator_names
+        @operator_names ||= operators.map(&:op_name)
+      end
+
+      def codeset_only?
+        return @codeset_only if defined?(@codeset_only)
+
+        @codeset_only = operators.reject do |op|
+          # We skip Unions because they don't count against codesets
+          op.op_name == "union"
+        end.map(&:class).map(&:basic_type).all? { |type| type == :selection }
+      end
+
+      def max_depth
+        operators.reverse.each.with_object({}) do |op, counts|
+          counts[op] = if op.all_upstreams.empty?
+                         1
+          else
+                         counts.values_at(*op.all_upstreams).max + 1
+          end
+        end.values.max
+      end
+
+      def num_selection_nodes
+        operators.select { |op| op.upstreams.empty? }.length
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/behaviors/costish.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "../../operators/vocabulary"
+
+module ConceptQL
+  module Vocabularies
+    module Behaviors
+      module Costish
+        def query(db)
+          costs = db[:procedure_cost].where(column_to_search => values).select(:procedure_occurrence_id)
+          db[:procedure_occurrence].where(procedure_occurrence_id: costs)
+        end
+
+        def table
+          :procedure_occurrence
+        end
+
+        def column_to_search
+          vocab_entry.id == "revenue_code" ? :revenue_code_source_value : :disease_class_source_value
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/behaviors/gdmish.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Vocabularies
+    module Behaviors
+      module Gdmish
+        def exclusion_clause(_db)
+          return {} unless select_all?
+
+          { clinical_code_concept_id: 0 }
+        end
+
+        def additional_columns(_db)
+          [ Sequel.cast_string(domain.to_s).as(:criterion_domain) ]
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/behaviors/omopish.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Vocabularies
+    module Behaviors
+      module Omopish
+        # Overrides Operator#tables
+        def tables
+          domains
+        end
+
+        def vocabulary_condition
+          { dm.source_vocabulary_id(domain) => vocabulary_id.to_i }
+        end
+
+        def where_clause(_db)
+          conds = vocabulary_condition
+          conds[dm.source_value_column(domain)] = arguments unless select_all?
+          conds
+        end
+
+        def query_cols
+          dm.table_columns(domain)
+        end
+
+        def translated_vocabulary_id
+          vocab_entry.omopv4_id
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/dynamic_vocabularies.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "csv"
+require_relative "entry"
+
+module ConceptQL
+  module Vocabularies
+    class DynamicVocabularies
+      # lexicon: the Lexicon whose vocabularies become operators, normally a
+      # ConceptQL::Database's own (Database#operators passes it). Without one,
+      # only the CSV vocabularies are used.
+      def initialize(lexicon = nil)
+        @lexicon = lexicon
+      end
+
+      # Vocabulary operators for one data model, as name => operator class.
+      # Fresh classes on every call.
+      def operators(data_model)
+        all_vocabs.each_with_object({}) do |(name, entry), h|
+          # A CSV row marked requires_lexicon (CPT4_HCPCS) supplies metadata
+          # only. Its operator exists only where the lexicon has the vocabulary.
+          next if entry.requires_lexicon? && !entry.from_lexicon?
+
+          klass = entry.dup.get_klasses[data_model]
+          h[name] = klass if klass
+        end
+      end
+
+      def all_vocabs
+        @all_vocabs ||= each_vocab.each_with_object({}) do |row, h|
+          entry = Entry.new(row.to_hash.compact)
+          h[entry.id] ||= entry
+          h[entry.id] = h[entry.id].merge(entry)
+        end
+      end
+
+      private
+
+      attr_reader :lexicon
+
+      def each_vocab
+        @each_vocab ||= get_all_vocabs
+      end
+
+      def get_all_vocabs
+        vocabs = [ ConceptQL.vocabularies_file_path,
+                  ConceptQL.custom_vocabularies_file_path ].select(&:exist?).map do |path|
+                   CSV.foreach(path, headers: true, header_converters: :symbol).to_a
+                 end.inject(:+).each do |v|
+          v[:from_csv] = true
+        end
+
+        lexicon_vocabularies + vocabs
+      end
+
+      # A Database accepts any object as its lexicon (opts[:lexicon]); one that
+      # keeps no vocabulary list contributes no vocabulary operators.
+      def lexicon_vocabularies
+        lexicon.respond_to?(:vocabularies) ? lexicon.vocabularies : []
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/entry.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require_relative "vocabulary_class_factory"
+
+module ConceptQL
+  module Vocabularies
+    class Entry
+      attr_reader :hash
+
+      METHODS = %i[
+        id
+        omopv4_vocabulary_id
+        omopv5_vocabulary_id
+        domain
+        vocabulary_short_name
+        vocabulary_long_name
+        format_regexp
+      ].freeze
+
+      METHODS.each do |meth|
+        define_method(meth) do
+          hash[meth]
+        end
+      end
+
+      METADATA_METHODS = METHODS + %i[
+        preferred_name
+        predominant_domains
+      ]
+
+      COST_RELATED_VOCABS = [
+        "Revenue Code",
+        "DRG"
+      ].freeze
+
+      def initialize(hash)
+        @hash = hash
+        @hash[:omopv5_vocabulary_id] ||= @hash[:id]
+        @hash[:aliases_arr] ||= []
+        @hash[:aliases_arr] |= (@hash[:aliases] || "").split(";")
+        @hash[:id] = @hash[:id].to_s.downcase
+        @hash[:aliases_arr] << @hash[:id] if @hash[:id] =~ /\s/
+        @hash[:id] = translate_id(@hash[:id].gsub(/\W+/, "_"))
+        @hash[:aliases_arr] -= [ @hash[:id] ]
+      end
+
+      def translate_id(id)
+        {
+          "icd10" => "icd10who"
+        }[id] || id
+      end
+
+      def merge(other_entry)
+        self.class.new(hash.merge(other_entry.hash))
+      end
+
+      def belongs_in_omopv4_plus?
+        @belongs_in_omopv4_plus ||= (!from_lexicon? || from_csv?) \
+          && has_domain? \
+          && visible?
+      end
+
+      def to_hash
+        METADATA_METHODS.to_h do |meth|
+          [ meth, send(meth) ]
+        end
+      end
+
+      def short_name
+        vocabulary_short_name || vocabulary_long_name
+      end
+
+      def long_name
+        vocabulary_long_name || vocabulary_short_name
+      end
+
+      def preferred_name
+        omopv5_id || id
+      end
+
+      def aliases
+        @hash[:aliases_arr]
+      end
+
+      def predominant_domains
+        Array(domain || :condition_occurrence).flatten
+      end
+
+      def from_lexicon?
+        hash[:from_lexicon].present?
+      end
+
+      def from_csv?
+        hash[:from_csv].present?
+      end
+
+      def requires_lexicon?
+        hash[:requires_lexicon].present?
+      end
+
+      def has_domain?
+        hash[:domain].present?
+      end
+
+      def hidden?
+        hash[:hidden].present?
+      end
+
+      def is_labish?
+        hash[:domain] =~ /observation|measurement/i
+      end
+
+      def is_drugish?
+        hash[:domain] =~ /drug_exposure/i
+      end
+
+      def is_costish?
+        belongs_in_omopv4_plus? && COST_RELATED_VOCABS.include?(omopv5_vocabulary_id)
+      end
+
+      def visible?
+        !hidden?
+      end
+
+      def omopv5_id
+        omopv5_vocabulary_id || id
+      end
+
+      def omopv4_id
+        omopv4_vocabulary_id
+      end
+
+      def get_klasses
+        VocabularyClassFactory.new(self).get_klasses
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/vocabulary_class_factory.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "../behaviors/drugish"
+require_relative "../behaviors/labish"
+require_relative "behaviors/gdmish"
+require_relative "behaviors/omopish"
+require_relative "behaviors/costish"
+require_relative "../operators/vocabulary"
+
+module ConceptQL
+  module Vocabularies
+    class VocabularyClassFactory
+      attr_reader :entry
+
+      def initialize(entry)
+        @entry = entry
+      end
+
+      def get_klasses
+        {
+          gdm: get_gdm_klass,
+          gdm_wide: get_gdm_klass,
+          omopv4_plus: get_omopv4_plus_klass
+        }.compact
+      end
+
+      def get_gdm_klass
+        get_klass do
+          include Behaviors::Gdmish
+        end
+      end
+
+      def get_omopv4_plus_klass
+        return nil unless entry.belongs_in_omopv4_plus?
+
+        get_klass do
+          include Behaviors::Omopish
+
+          include Behaviors::Costish if entry.is_costish?
+        end
+      end
+
+      def get_klass(&block)
+        ventry = entry
+        Class.new(ConceptQL::Operators::Vocabulary) do |klass|
+          @entry = ventry
+
+          class << self
+            attr_reader :entry
+          end
+
+          preferred_name entry.preferred_name
+          argument :codes, type: :codelist
+          aliases entry.aliases
+          predominant_domains entry.predominant_domains
+          short_name entry.short_name
+          long_name entry.long_name
+          conceptql_spec_id "vocabulary"
+
+          klass.class_eval(&block)
+
+          def self.inspect
+            name
+          end
+
+          def self.to_s
+            name
+          end
+
+          def self.name
+            "ConceptQL::Operator::#{entry.id}"
+          end
+
+          include ConceptQL::Behaviors::Labish if entry.is_labish?
+
+          include ConceptQL::Behaviors::Drugish if entry.is_drugish?
+
+          def vocab_entry
+            self.class.entry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/window/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Window
+    # Base class for classes apply scope windows to incoming Datasets
+    class Base
+      attr_reader :opts
+
+      def initialize(opts = {})
+        @opts = opts
+      end
+
+      def call(_op, _ds, _options = {})
+        raise NotImplementedError
+      end
+
+      def event_start_date_column
+        Sequel[opts[:event_start_date_column].to_sym]
+      end
+
+      def event_end_date_column
+        Sequel[opts[:event_end_date_column].to_sym]
+      end
+    end
+  end
+end

data/lib/conceptql/window/date_range.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module ConceptQL
+  module Window
+    # Provides a scope window that uses a date range literal
+    class DateRange < Base
+      def call(operator, dset, options = {})
+        return dset if options[:timeless]
+
+        rdbms = operator.rdbms
+        dset.where(start_check(rdbms)).where(end_check(rdbms)).from_self
+      end
+
+      def start_check(rdbms)
+        rdbms.cast_date(opts[:start_date]) <= event_start_date_column
+      end
+
+      def end_check(rdbms)
+        event_end_date_column <= rdbms.cast_date(opts[:end_date])
+      end
+    end
+  end
+end