conceptql 0.3.0 → 0.4.1

data/lib/conceptql/operators/vocabulary.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Vocabulary < Operator
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::CodeLister
+
+      category "Select by Clinical Codes"
+      basic_type :selection
+      desc "Selects records based on the given vocabulary."
+      validate_no_upstreams
+      validate_at_least_one_argument
+      validate_codes_match
+
+      def source_table
+        domain_map(op_name)
+      end
+
+      def table
+        domain_map(op_name)
+      end
+
+      def domain
+        domain_map(op_name)
+      end
+
+      def domains(_db)
+        vocab_entry.predominant_domains.map(&:to_sym)
+      end
+
+      def additional_validation(db, opts = {})
+        return unless add_warnings?(db, opts) && !select_all?
+
+        args = arguments.dup
+        args -= bad_arguments
+
+        missing_args = args - dm.known_codes(db, vocabulary_id, args)
+
+        return if missing_args.empty?
+
+        add_warning("unknown code(s)", *missing_args)
+      end
+
+      def describe_codes(db, codes)
+        return [ [ "*", "ALL CODES" ] ] if select_all?
+
+        found_codes = dm.concepts_to_codes(db, vocabulary_id, codes)
+
+        found_codes + (codes - found_codes.map(&:first)).zip([])
+      end
+
+      def filter_clause(db)
+        filter_clause = Sequel[where_clause(db)]
+        if (ex_clause = exclusion_clause(db)).present?
+          filter_clause &= ~Sequel[ex_clause]
+        end
+        filter_clause
+      end
+
+      def criterion_table
+        dm.table_by_domain(domain)
+      end
+
+      # The condition that selects this vocabulary's rows. where_clause
+      # includes it, so Union can name every merged vocabulary in one IN list.
+      def vocabulary_condition
+        { clinical_code_vocabulary_id: vocabulary_id }
+      end
+
+      private
+
+      def query(db)
+        ds = db[criterion_table]
+
+        ds = ds.where(filter_clause(db))
+
+        unless (more_columns = additional_columns(db)).empty?
+          ds = ds.select_append(*more_columns)
+        end
+
+        ds
+      end
+
+      def exclusion_clause(_db)
+        {}
+      end
+
+      def additional_columns(_db)
+        []
+      end
+
+      def where_clause(db)
+        # We're probably going to partition the observations table by clinical_code_vocabulary_id,
+        # for Spark, so this will create queries that leverage that partitioning
+        # and it shouldn't hurt for PostgreSQL performance
+        wheres = vocabulary_condition
+
+        unless select_all?
+          concept_ids = dm.concept_ids(db, vocabulary_id, arguments)
+
+          wheres = wheres.merge({ clinical_code_concept_id: concept_ids })
+        end
+
+        wheres
+      end
+
+      def query_cols
+        dm.table_columns(source_table)
+      end
+
+      def select_all?
+        arguments.include?("*")
+      end
+
+      def preferred_name
+        vocab_entry.short_name || vocab_entry.omopv5_id
+      end
+
+      # Defined so that bad_arguments can check for bad codes
+      def code_regexp
+        unless defined?(@code_regexp)
+          @code_regexp = nil
+
+          if (reg_str = vocab_entry.format_regexp)
+            @code_regexp = Regexp.union(Regexp.new(reg_str, Regexp::IGNORECASE), /\A\*\Z/)
+          end
+        end
+        @code_regexp
+      end
+
+      def vocabulary_id
+        @vocabulary_id ||= translated_vocabulary_id
+      end
+
+      def translated_vocabulary_id
+        vocab_entry.omopv5_vocabulary_id || op_name
+      end
+
+      def domain_map(_v_id)
+        (vocab_entry.domain || :condition_occurrence).to_sym
+      end
+
+      def table_is_missing?(db)
+        dm.table_is_missing?(db)
+      end
+    end
+  end
+end

data/lib/conceptql/paths.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Pathname is core only from Ruby 4.0; on 3.x a clean `gem install` (no Bundler
+# preloading it) fails in ConceptQL.root without this. RuboCop targets 4.0.
+require "pathname"
+
+module ConceptQL
+  class << self
+    def root
+      Pathname.new(__dir__).parent.parent.expand_path
+    end
+
+    def schemas_dir
+      root / "schemas"
+    end
+
+    def config_dir
+      root / "config"
+    end
+
+    def vocabularies_file_path
+      ConceptQL.config_dir / "vocabularies.csv"
+    end
+
+    def multiple_vocabularies_file_path
+      ConceptQL.config_dir / "multiple_vocabularies.csv"
+    end
+
+    def custom_vocabularies_file_path
+      Pathname.new(ENV["CONCEPTQL_CUSTOM_VOCABULARIES"] || ConceptQL.config_dir / "vocabularies.custom.csv")
+    end
+
+    def custom_multiple_vocabularies_file_path
+      Pathname.new(ENV["CONCEPTQL_CUSTOM_MULTIPLE_VOCABULARIES"] || ConceptQL.config_dir / "multiple_vocabularies.custom.csv")
+    end
+
+    def race_file
+      ConceptQL.config_dir / "race.yml"
+    end
+  end
+end

data/lib/conceptql/query.rb

@@ -1,96 +1,218 @@
-require 'json'
-require 'open3'
-require 'forwardable'
-require_relative 'scope'
-require_relative 'nodifier'
-require_relative 'sql_formatter'
+# frozen_string_literal: true
+
+require "json"
+require "forwardable"
+require_relative "scope"
+require_relative "nodifier"
+require "sequel/sqlformat"
 
 module ConceptQL
   class Query
+    class QueryError < StandardError
+      def message
+        return super unless cause
+
+        og_message = if cause.respond_to?(:full_message)
+                       cause.full_message
+        else
+                       ([ cause.message ] + cause.backtrace).join("\n")
+        end
+        [ super, "OG ERROR:", og_message ].join("\n")
+      end
+    end
+
+    # Raised by #code_list when the statement has validation errors: the
+    # operator tree of an invalid statement can have missing operands, and a
+    # partial code list would look complete. #errors is the errors hash of
+    # #scope_annotate (operator name => list of [message, *details]).
+    class InvalidStatementError < QueryError
+      attr_reader :statement, :errors
+
+      def initialize(statement, errors)
+        @statement = statement
+        @errors = errors
+        super([ "Invalid ConceptQL statement, so it has no code list. Validation errors:",
+               *errors.flat_map { |op, op_errors| op_errors.map { |e| "  #{op}: #{describe(e)}" } },
+               "Statement: #{statement.inspect}" ].join("\n"))
+      end
+
+      private
+
+      def describe(error)
+        message, *details = error
+        details.empty? ? message.to_s : "#{message} (#{details.flatten.map(&:to_s).join(', ')})"
+      end
+    end
+
     extend Forwardable
-    def_delegators :query, :all, :count, :execute, :order, :profile
-    def_delegators :db, :profile_for
-
-    attr :statement
-    def initialize(db, statement, opts={})
-      @db = db
-      @statement = extract_statement(statement)
-      opts = opts.dup
+    def_delegators :query, :all, :count, :execute, :order
+    def_delegators :cdb, :db
+
+    attr_reader :statement
+
+    def initialize(cdb, statement, opts = {})
+      @cdb = cdb
+      @statement, opts = extract_statement(statement, opts)
+      @opts = opts
       opts[:algorithm_fetcher] ||= proc do |alg|
-        statement, description = db[:concepts].where(concept_id: alg).get([:statement, :label])
+        statement, description = db[:concepts].where(concept_id: alg).get(%i[statement label])
         statement = JSON.parse(statement) if statement.is_a?(String)
-        [statement, description]
+        [ statement, description ]
       end
-      @nodifier = opts[:nodifier] || Nodifier.new({ database_type: db.database_type}.merge(opts))
+      @nodifier = opts[:nodifier] || Nodifier.new(cdb, { database_type: cdb.database_type }.merge(opts))
     end
 
-    def query
-      nodifier.scope.with_ctes(operator.evaluate(db), db)
+    def analyze
+      query(explain: true, analyze: true).analyze
+    end
+
+    def query(opts = {})
+      (@queries ||= {})[opts] ||= nodifier.scope.with_ctes(operator, db, opts)
     end
 
     def query_cols(opts = {})
       cols = operator.dynamic_columns
       if opts[:cast]
-        cols = query_cols.each_with_object({}) do |column, h|
-          h[column] = operator.cast_column(column)
+        cols = query_cols.to_h do |column|
+          [ column, operator.cast_column(column) ]
         end
       end
       cols
     end
 
-    def sql
-      SqlFormatter.new.format(query.sql)
-    rescue
-      puts $!.message
-      puts $!.backtrace.join("\n")
-      return "SQL unavailable for this statement"
+    def column_names
+      query_cols.is_a?(Hash) ? query_cols.keys : query_cols
+    end
+
+    def sql(*args)
+      sql_statements(*args).values.join(";\n")
+    end
+
+    def sql_statements(*args)
+      stmts = query.sql_statements
+
+      if args.include?(:create_tables)
+        sql = stmts.delete(:query)
+        drop_stmts = []
+        if args.include?(:drop_tables)
+          drop_stmts = stmts.map do |name, _|
+            [ [ "drop", name ].join("_"), db.send(:drop_table_sql, name, if_exists: true) ]
+          end
+        end
+        create_stmts = stmts.map do |name, sql|
+          [ name, db.send(:create_table_as_sql, name, sql, {}) ]
+        end
+        stmts = drop_stmts + create_stmts
+        stmts.push([ :query, sql ])
+      end
+
+      if args.include?(:formatted)
+        stmts = stmts.map do |name, sql|
+          [ name, format(sql) ]
+        end
+      end
+      Hash[stmts]
+    rescue StandardError
+      raise QueryError, "Failed to generate SQL for #{statement.inspect}"
     end
 
     def annotate(opts = {})
       operator.annotate(db, opts)
     end
 
+    def accept(visitor, _opts = {})
+      operator.accept(visitor)
+    end
+
+    def extract_metadata
+      visitor = ConceptQL::Visitors::MetadataExtractor.new
+      operator.accept(visitor)
+      visitor.results
+    end
+
     def scope_annotate(opts = {})
       annotate(opts)
       nodifier.scope.annotation
     end
 
-    def optimized
-      n = dup
-      n.instance_variable_set(:@operator, operator.optimized)
-      n
-    end
-
     def domains
       operator.domains(db)
     end
 
+    def events_per_patient
+      operator.events_per_patient
+    end
+
+    def multiple_vocabularies
+      operator.multiple_vocabularies
+    end
+
     def operator
       @operator ||= if statement.is_a?(Array)
-        if statement.first.is_a?(Array)
-          Operators::Invalid.new(nodifier, "invalid", errors: [["incomplete statement"]])
-        else
-          nodifier.create(*statement)
-        end
+                      if statement.first.is_a?(Array)
+                        Operators::Invalid.new(nodifier, "invalid", errors: [ [ "incomplete statement" ] ])
+                      else
+                        nodifier.create(*statement)
+                      end
       else
-        Operators::Invalid.new(nodifier, "invalid", errors: [["invalid root operator", statement.inspect]])
+                      Operators::Invalid.new(nodifier, "invalid",
+                                             errors: [ [ "invalid root operator", statement.inspect ] ])
       end
     end
 
-    def code_list(db)
-      operator.code_list(db)
+    def rdbms
+      operator.rdbms
+    end
+
+    # Raises InvalidStatementError if the statement has validation errors.
+    # Warnings do not raise.
+    def code_list(_ignored_db = nil)
+      errors = validation_errors
+      raise InvalidStatementError.new(statement, errors) unless errors.empty?
+
+      operator.code_list(db).uniq
+    end
+
+    # The errors that #scope_annotate reports, the same check that callers use
+    # to decide whether a statement is valid. A fresh query runs it, because
+    # operators memoize #annotate and #validate, and this query's later
+    # #annotate calls must not get those results.
+    def validation_errors
+      Query.new(cdb, statement, @opts).scope_annotate(skip_counts: true)[:errors]
+    end
+
+    def drop_temp_tables(opts = {})
+      query.drop_temp_tables(opts)
     end
 
     private
-    attr :db, :nodifier
 
+    attr_reader :cdb, :nodifier
+
+    def extract_statement(stmt, opts)
+      if !stmt.is_a?(Array)
+        raise "Improper ConceptQL statement: Expected an Array, got a #{stmt.class}"
+      elsif stmt.first.to_s == "window"
+        raise "window operator needs a hash as the last item of the array." unless stmt.last.is_a?(Hash)
+        unless stmt.length == 3
+          raise "window operator needs a ConceptQL statement followed by a hash as the last item of the array."
+        end
 
-    def extract_statement(stmt)
-      if stmt.is_a?(Array) && stmt.length == 1 && stmt.first.is_a?(Array)
-        stmt.first
+        opts[:scope_opts] = (opts[:scope_opts] || {})
+        opts[:scope_opts].merge!(window_opts: stmt.last.merge(cdb: cdb))
+        extract_statement(stmt[1], opts)
+      elsif stmt.length == 1 && stmt.first.is_a?(Array)
+        extract_statement(stmt.first, opts)
       else
-        stmt
+        [ stmt, opts ]
       end
     end
+
+    # Formats inside the Ruby process with the sequel-sqlformat gem (the Rust
+    # sqlformat crate). It changes only whitespace and keyword case, and it never
+    # fails on unknown syntax, so DuckDB and Spark SQL format too.
+    def format(sql)
+      Sequel::SQLFormat.format(sql, dialect: Sequel::SQLFormat.dialect_for(cdb.database_type), indent: 2, uppercase: true)
+    end
   end
 end

data/lib/conceptql/query_modifiers/gdm/admission_date_query_modifier.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Gdm
+      class AdmissionDateQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            admission_date
+            discharge_date
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :collection_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:collection_id)
+
+          query.from_self(alias: :cc)
+               .left_join(:collections, { id: :collection_id }, table_alias: :co)
+               .left_join(:admission_details, { id: :admission_detail_id }, table_alias: :ad)
+               .select_all(:cc)
+               .select_append(Sequel.function(:coalesce, Sequel[:ad][:admission_date],
+                                              Sequel[:cc][:start_date]).as(:admission_date))
+               .select_append(Sequel.function(:coalesce, Sequel[:ad][:discharge_date], Sequel[:cc][:end_date],
+                                              Sequel[:cc][:start_date]).as(:discharge_date))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm/drug_query_modifier.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Gdm
+      class DrugQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            drug_name
+            drug_amount
+            drug_amount_units
+            drug_quantity
+            drug_strength_source_value
+            drug_days_supply
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :drug_exposure_detail_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:drug_exposure_detail_id)
+
+          # TODO: Determine what actual columns to include for drug exposures under
+          query.from_self(alias: :cc)
+               .left_join(Sequel[:drug_exposure_details].as(:de), Sequel[:cc][:drug_exposure_detail_id] => Sequel[:de][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:dose_con), Sequel[:de][:dose_unit_concept_id] => Sequel[:dose_con][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:ing_con), Sequel[:cc][:clinical_code_concept_id] => Sequel[:ing_con][:id])
+               .select_all(:cc)
+               .select_append(Sequel[:de][:dose_value].as(:drug_amount))
+               .select_append(Sequel[:dose_con][:concept_text].as(:drug_amount_units))
+               .select_append(Sequel[:ing_con][:concept_text].as(:drug_name))
+               .select_append(Sequel[:de][:days_supply].as(:drug_days_supply))
+               .select_append(Sequel[:de][:strength_source_value].as(:drug_strength_source_value))
+               .select_append(Sequel[:cc][:quantity].as(:drug_quantity))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm/lab_query_modifier.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Gdm
+      class LabQueryModifier < QueryModifier
+        def self.provided_columns
+          %i[
+            value_as_number
+            value_as_string
+            value_as_concept_id
+            value_as_concept
+            unit_concept_id
+            unit_concept
+            range_low
+            range_high
+          ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :measurement_detail_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(:measurement_detail_id)
+
+          query.from_self(alias: :cc)
+               .left_join(Sequel[:measurement_details].as(:md), Sequel[:cc][:measurement_detail_id] => Sequel[:md][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:unit_con), Sequel[:md][:unit_concept_id] => Sequel[:unit_con][:id])
+               .left_join(dm.name_concepts_table(query.db).as(:concept_con), Sequel[:md][:result_as_concept_id] => Sequel[:concept_con][:id])
+               .select_all(:cc)
+               .select_append(Sequel[:md][:result_as_number].as(:value_as_number))
+               .select_append(Sequel[:md][:result_as_string].as(:value_as_string))
+               .select_append(Sequel[:md][:result_as_concept_id].as(:value_as_concept_id))
+               .select_append(Sequel[:concept_con][:concept_code].as(:value_as_concept))
+               .select_append(Sequel[:md][:unit_concept_id].as(:unit_concept_id))
+               .select_append(Sequel[:unit_con][:concept_code].as(:unit_concept))
+               .select_append(Sequel[:md][:normal_range_low].as(:range_low))
+               .select_append(Sequel[:md][:normal_range_high].as(:range_high))
+               .from_self
+        end
+
+        private
+
+        def domain
+          op.domain
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/query_modifiers/gdm/pos_query_modifier.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../query_modifier"
+
+module ConceptQL
+  module QueryModifiers
+    module Gdm
+      class PoSQueryModifier < QueryModifier
+        attr_reader :join_id, :table, :source_column, :column
+
+        def self.provided_columns
+          [ :visit_source_concept_id ]
+        end
+
+        def self.has_required_columns?(cols)
+          needed = [ :context_id ]
+          found = needed & cols
+          needed == found
+        end
+
+        def initialize(*args)
+          super
+          @column = :visit_source_concept_id
+          @join_id = :context_id
+          @table = :contexts
+          @source_column = :pos_concept_id
+        end
+
+        def modified_query
+          return query unless dm.table_cols(source_table).include?(join_id)
+
+          left_alias = "tab1".to_sym
+          right_alias = "tab2".to_sym
+
+          extra_table = query.db.from(table).select(Sequel.as(source_column, column), :id)
+          ds = query.from_self(alias: left_alias)
+          ds.left_join(extra_table.as(right_alias),
+                       Sequel.qualify(left_alias, join_id) => Sequel.qualify(right_alias, :id))
+            .select_all(left_alias)
+            .select_append(Sequel.qualify(right_alias, column).as(column))
+            .from_self
+        end
+      end
+    end
+  end
+end