conceptql 0.2.0 → 0.4.0

data/lib/conceptql/query_modifiers/gdm/provider_query_modifier.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Gdm
+      class ProviderQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[provider_id context_specialty_concept_id practitioner_specialty_concept_id role_type_concept_id]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = %i[practitioner_id context_id].sort
+          found = needed | cols
+          !found.empty?
+        end
+
+        def modified_query
+          if dm.table_cols(source_table).include?(:context_id)
+            query.from_self(alias: :c)
+                 .left_join(Sequel[:contexts_practitioners].as(:cp), Sequel[:c][:context_id] => Sequel[:cp][:context_id])
+                 .left_join(Sequel[:practitioners].as(:p), Sequel[:p][:id] => Sequel[:cp][:practitioner_id])
+                 .select_all(:c)
+                 .select_append(
+                   Sequel[:cp][:practitioner_id].as(:provider_id),
+                   Sequel[:cp][:role_type_concept_id].as(:role_type_concept_id),
+                   Sequel[:cp][:specialty_type_concept_id].as(:context_specialty_concept_id),
+                   Sequel[:p][:specialty_concept_id].as(:practitioner_specialty_concept_id)
+                 )
+          else
+            ds = query.select_all
+            ds = if ds.columns.include?(:provider_id)
+                   ds
+            elsif dm.table_cols(source_table).include?(:practitioner_id)
+                   ds.select_append(Sequel[:practitioner_id].as(:provider_id))
+            else
+                   ds.select_append(Sequel.cast(nil, :Bigint).as(:provider_id))
+            end
+            %i[role_type_concept_id context_specialty_concept_id practitioner_specialty_concept_id].each do |col|
+              ds = if ds.columns.include?(col)
+                     ds
+              else
+                     ds.select_append(Sequel.cast(nil, :Bigint).as(col))
+              end
+            end
+            ds
+          end.from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm_wide/admission_date_query_modifier.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module GdmWide
+      class AdmissionDateQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            admission_date
+            discharge_date
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = %i[admit_admission_date admit_discharge_date].sort
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:admit_admission_date)
+
+          query.from_self
+               .select_append(Sequel.function(:coalesce, :admit_admission_date, :start_date).as(:admission_date))
+               .select_append(Sequel.function(:coalesce, :admit_discharge_date, :end_date,
+                                              :start_date).as(:discharge_date))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm_wide/drug_query_modifier.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module GdmWide
+      class DrugQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            drug_name
+            drug_amount
+            drug_amount_units
+            drug_quantity
+            drug_strength_source_value
+            drug_days_supply
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :dose_unit_concept_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:dose_unit_concept_id)
+
+          query.from_self(alias: :cc)
+               .left_join(dm.name_concepts_table(query.db).as(:dose_con), Sequel[:cc][:dose_unit_concept_id] => Sequel[:dose_con][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:ing_con), Sequel[:cc][:clinical_code_concept_id] => Sequel[:ing_con][:id])
+               .select_all(:cc)
+               .select_append(Sequel[:cc][:dose_value].as(:drug_amount))
+               .select_append(Sequel[:dose_con][:concept_text].as(:drug_amount_units))
+               .select_append(Sequel[:ing_con][:concept_text].as(:drug_name))
+               .select_append(Sequel[:cc][:days_supply].as(:drug_days_supply))
+               .select_append(Sequel[:cc][:strength_source_value].as(:drug_strength_source_value))
+               .select_append(Sequel[:cc][:quantity].as(:drug_quantity))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm_wide/lab_query_modifier.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module GdmWide
+      class LabQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            value_as_number
+            value_as_string
+            value_as_concept_id
+            value_as_concept
+            unit_concept_id
+            unit_concept
+            range_low
+            range_high
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :unit_concept_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:unit_concept_id)
+
+          query.from_self(alias: :cc)
+               .left_join(dm.name_concepts_table(query.db).as(:unit_con), Sequel[:cc][:unit_concept_id] => Sequel[:unit_con][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:concept_con), Sequel[:cc][:result_as_concept_id] => Sequel[:concept_con][:id])
+               .select_all(:cc)
+               .select_append(Sequel[:cc][:result_as_number].as(:value_as_number))
+               .select_append(Sequel[:cc][:result_as_string].as(:value_as_string))
+               .select_append(Sequel[:cc][:result_as_concept_id].as(:value_as_concept_id))
+               .select_append(Sequel[:concept_con][:concept_code].as(:value_as_concept))
+               .select_append(Sequel[:unit_con][:concept_code].as(:unit_concept))
+               .select_append(Sequel[:cc][:normal_range_low].as(:range_low))
+               .select_append(Sequel[:cc][:normal_range_high].as(:range_high))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm_wide/pos_query_modifier.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module GdmWide
+      class PoSQueryModifier < QueryModifier
+        attr_reader :source_column, :column
+
+        def self.provided_columns
+          [ :visit_source_concept_id ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :context_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def initialize(*args)
+          super
+          @column = :visit_source_concept_id
+          @source_column = :pos_concept_id
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:context_id)
+
+          query.from_self
+               .select_append(Sequel[source_column].as(column))
+               .from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm_wide/provenance_query_modifier.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module GdmWide
+      class ProvenanceQueryModifier < QueryModifier
+        RELATED_COLUMNS = %w[
+          provenance_concept_id
+          source_type_concept_id
+        ].sort.map(&:to_sym)
+
+        attr_reader :db
+
+        def self.provided_columns
+          %i[
+            file_provenance_type
+            code_provenance_type
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          !(RELATED_COLUMNS & cols).empty?
+        end
+
+        def modified_query
+          return query unless self.class.has_required_columns?(dm.table_cols(source_table))
+
+          query.from_self
+               .select_append(Sequel[:source_type_concept_id].as(:file_provenance_type),
+                              Sequel[:provenance_concept_id].as(:code_provenance_type))
+               .from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Omopv4Plus
+      class AdmissionDateQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            admission_date
+            discharge_date
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          cols.map(&:to_s).any? { |col| col =~ /start_date/i }
+        end
+
+        def modified_query
+          cols = dm.table_cols(source_table).map(&:to_s)
+          return query unless cols.any? { |col| col =~ /start_date/i }
+
+          start_col = cols.find { |col| col =~ /start_date/i }.to_sym
+          end_col = cols.find { |col| col =~ /end_date/i }.to_sym
+
+          query.from_self(alias: :t)
+               .select_all(:t)
+               .select_append(Sequel[:t][start_col].as(:admission_date))
+               .select_append(Sequel.function(:coalesce, Sequel[:t][end_col],
+                                              Sequel[:t][start_col]).as(:discharge_date))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Omopv4Plus
+      class DrugQueryModifier < QueryModifier
+        attr_reader :db
+
+        def self.provided_columns
+          %i[
+            drug_name
+            drug_amount
+            drug_amount_units
+            drug_quantity
+            drug_days_supply
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :drug_concept_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def initialize(*args)
+          super
+          @db = query.db
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:drug_concept_id)
+
+          query.db
+          drug_concept_ids = query.select(:drug_concept_id).distinct
+          query.from_self(alias: :de)
+               .left_join(micro_table(drug_concept_ids).as(:mt), drug_concept_id: :drug_concept_id)
+               .select_all(:de)
+               .select_append(Sequel[:de][:quantity].as(:drug_quantity))
+               .select_append(Sequel[:de][:days_supply].as(:drug_days_supply))
+               .select_append(Sequel[:mt][:amount_value].as(:drug_amount))
+               .select_append(Sequel[:mt][:amount_unit].as(:drug_amount_units))
+               .select_append(Sequel[:mt][:drug_name].as(:drug_name))
+               .from_self
+        end
+
+        private
+
+        def micro_table(drug_concept_ids)
+          # TODO: Does drug_strength only have RXNORM concept_ids?
+          # TODO: What is vocabulary for units?  Can we shrink concept table to just that vocab before joining?
+          collapsed_strengths = db[:drug_strength]
+                                .where(drug_concept_id: drug_concept_ids)
+                                .select_group(:drug_concept_id)
+                                .select_append(
+                                  Sequel.function(:min, :amount_value).as(:amount_value),
+                                  Sequel.function(:min, :amount_unit).as(:amount_unit),
+                                  Sequel.function(:count, 1).as(:dcount)
+                                )
+
+          value_case = Sequel.case({ 1 => :amount_value }, nil, :dcount).as(:amount_value)
+          unit_case = Sequel.case({ 1 => :amount_unit }, nil, :dcount).as(:amount_unit)
+
+          collapsed_strengths = collapsed_strengths.from_self.select(:drug_concept_id, value_case, unit_case)
+
+          db.from(Sequel[:concept].as(:dc))
+            .left_join(collapsed_strengths, { drug_concept_id: :concept_id }, { table_alias: :ds })
+            .where(Sequel[:dc][:concept_id] => drug_concept_ids)
+            .select(
+              Sequel[:dc][:concept_id].as(:drug_concept_id),
+              Sequel[:dc][:concept_name].as(:drug_name),
+              Sequel[:ds][:amount_value].as(:amount_value),
+              Sequel[:ds][:amount_unit].as(:amount_unit)
+            )
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Omopv4Plus
+      class PoSQueryModifier < QueryModifier
+        attr_reader :join_id, :table, :source_column, :column
+
+        def self.provided_columns
+          [ :visit_source_concept_id ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :visit_occurrence_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def initialize(*args)
+          super
+          @column = :visit_source_concept_id
+          @join_id = :visit_occurrence_id
+          @table = :visit_occurrence
+          @source_column = :visit_source_concept_id
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(join_id)
+          return query if dm.table_cols(source_table).include?(column)
+
+          left_alias = "tab1".to_sym
+          right_alias = "tab2".to_sym
+
+          extra_table = query.db.from(table).select(Sequel.as(source_column, column), join_id)
+          ds = query.from_self(alias: left_alias)
+          ds.left_join(extra_table.as(right_alias),
+                       Sequel.qualify(left_alias, join_id) => Sequel.qualify(right_alias, join_id))
+            .select_all(left_alias)
+            .select_append(Sequel.qualify(right_alias, column).as(column))
+            .from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Omopv4Plus
+      class ProvenanceQueryModifier < QueryModifier
+        RELATED_COLUMNS = %w[
+          condition_type_concept_id
+          procedure_type_concept_id
+          death_type_concept_id
+          observation_type_concept_id
+          drug_type_concept_id
+        ].sort.map(&:to_sym)
+
+        attr_reader :db
+
+        def self.provided_columns
+          %i[
+            file_provenance_type
+            code_provenance_type
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          !(RELATED_COLUMNS & cols).empty?
+        end
+
+        def initialize(*args)
+          super
+          @db = query.db
+        end
+
+        def modified_query
+          return query unless self.class.has_required_columns?(dm.table_cols(source_table))
+
+          query.select_append(Sequel[dm.file_provenance_type_column(query, source_table)].as(:file_provenance_type),
+                              Sequel[dm.code_provenance_type_column(query,
+                                                                    source_table)].as(:code_provenance_type)).from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Omopv4Plus
+      class ProviderQueryModifier < QueryModifier
+        def self.provided_columns
+          [ :provider_id ]
+        end
+
+        def self.has_required_columns?(cols)
+          cols.find { |col| col.to_s =~ /provider_id/ }
+        end
+
+        def modified_query
+          col = self.class.has_required_columns?(dm.table_cols(source_table))
+          return query if col.nil? || col.to_sym == :provider_id
+
+          query.select_append(Sequel[col].as(:provider_id)).from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/query_modifier.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module QueryModifiers
+    class QueryModifier
+      attr_reader :query, :op, :source_table, :dm
+
+      def initialize(query, op, table, dm)
+        @query = query
+        @op = op
+        @source_table = table
+        @dm = dm
+      end
+
+      def modified_query
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/conceptql/rdbms/duckdb.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "generic"
+
+module ConceptQL
+  module Rdbms
+    class DuckDB < Generic
+      # Returns from - to (see Generic#datediff). DuckDB's
+      # datediff(part, startdate, enddate) returns enddate - startdate, so the
+      # arguments go in reversed.
+      def datediff(from, to)
+        Sequel.function(:datediff, "day", to, from)
+      end
+
+      def preferred_formatter
+        SqlFormatters::PG_FORMAT
+      end
+
+      # DuckDB accepts WITH ... AS MATERIALIZED since v0.8.0.
+      def supports_materialized?
+        true
+      end
+    end
+  end
+end

data/lib/conceptql/rdbms/generic.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Rdbms
+    class Generic
+      def process(column, value = nil)
+        type = Scope::COLUMN_TYPES.fetch(column)
+        new_column = case type
+        when String, :String
+                       Sequel.cast_string(value)
+        when Date, :Date
+                       cast_date(value)
+        when Float, :Bigint, :Float
+                       Sequel.cast_numeric(value, type)
+        else
+                       raise "Unexpected type: '#{type.inspect}' for column: '#{column}'"
+        end
+        new_column.as(column)
+      end
+
+      def semi_join(ds, table, *exprs)
+        ds = Sequel[ds] if ds.is_a?(Symbol)
+        table = Sequel[table] if table.is_a?(Symbol)
+        expr = exprs.inject(&:&)
+        ds.from_self(alias: :l).where(ds.db[table.as(:r)]
+          .select(1)
+          .where(expr)
+          .exists).from_self
+      end
+
+      # True when lhs and rhs are equal or both are NULL. IS NOT DISTINCT
+      # FROM is standard SQL, and Postgres and DuckDB support it. Spark
+      # overrides this with its <=> operator.
+      def null_safe_equal(lhs, rhs)
+        Sequel.lit("(? IS NOT DISTINCT FROM ?)", lhs, rhs)
+      end
+
+      def cast_date(date)
+        Sequel.cast(date, Date)
+      end
+
+      # The number of days from from_column to to_column: to_column - from_column.
+      # Note that datediff takes its arguments in the opposite order.
+      def days_between(from_column, to_column)
+        cast_date(to_column) - cast_date(from_column)
+      end
+
+      def uuid
+        uuid_items
+          .zip([ Sequel.cast_string("/") ] * (uuid_items.length - 1))
+          .flatten
+          .compact
+          .inject(:+)
+      end
+
+      def uuid_items
+        %w[person_id criterion_id criterion_table start_date end_date].map do |column|
+          Sequel.cast_string(column.to_sym)
+        end
+      end
+
+      def preferred_formatter
+        SqlFormatters::SQL_FORMATTER
+      end
+
+      # Returns from - to in days, NOT to - from, so datediff(later, earlier)
+      # is positive. Every adapter keeps this order. This default emits
+      # Spark's datediff(endDate, startDate), which returns endDate - startDate,
+      # and Spark uses it as is. days_between(from, to) returns to - from.
+      def datediff(from, to)
+        Sequel.function(:datediff, from, to)
+      end
+
+      def analyze_temp_tables?
+        false
+      end
+
+      def explain_temp_tables?
+        false
+      end
+
+      def supports_materialized?
+        false
+      end
+
+      def create_options(_scope, _ds)
+        opts = {}
+        opts[:analyze] = opts[:explain] = explain_temp_tables?
+        opts
+      end
+
+      def drop_options
+        {}
+      end
+
+      def post_create(db, table_name); end
+
+      # One primary clinical code per collection: the row with the lowest
+      # concept id. ROW_NUMBER() works on every supported RDBMS. Postgres
+      # overrides this with DISTINCT ON.
+      def primary_concepts(db, all_primary_ids)
+        db[Sequel[:clinical_codes].as(:pcc)]
+          .where(provenance_concept_id: all_primary_ids)
+          .select(
+            Sequel[:pcc][:collection_id].as(:collection_id),
+            Sequel[:pcc][:clinical_code_concept_id].as(:concept_id),
+            Sequel[:pcc][:clinical_code_source_value].as(:concept_code),
+            Sequel[:pcc][:clinical_code_vocabulary_id].as(:vocabulary_id),
+            Sequel[true].as(:is_primary)
+          )
+          .select_append(Sequel[:ROW_NUMBER].function.over(
+            partition: :collection_id,
+            order: [ Sequel[:pcc][:collection_id], Sequel[:pcc][:clinical_code_concept_id] ]
+          ).as(:nummy))
+          .from_self
+          .where(nummy: 1)
+      end
+    end
+  end
+end