conceptql 0.2.0 → 0.4.0

data/lib/conceptql/operators/operator.rb

@@ -1,235 +1,681 @@
-require 'zlib'
-require_relative '../behaviors/metadatable'
-require 'facets/array/extract_options'
-require 'facets/hash/deep_rekey'
-require 'forwardable'
+# frozen_string_literal: true
+
+require "zlib"
+require_relative "../behaviors/metadatable"
+require "forwardable"
 
 module ConceptQL
   module Operators
+    # Built-in operators only: the classes that call Operator.register when
+    # their file loads. Vocabulary operators are not in here; each
+    # ConceptQL::Database builds its own from its lexicon (Database#operators).
+    STATIC_OPERATORS = { omopv4_plus: {}, gdm: {}, gdm_wide: {} }.freeze
+
+    SELECTED_COLUMNS = %i[person_id
+                          criterion_id
+                          criterion_table
+                          criterion_domain
+                          start_date
+                          end_date
+                          value_as_number
+                          value_as_string
+                          value_as_concept_id
+                          units_source_value
+                          source_value].freeze
+
+    def self.static_operators
+      STATIC_OPERATORS
+    end
+
     class Operator
       extend Forwardable
-      extend Metadatable
-      COLUMNS = [
-        :person_id,
-        :criterion_id,
-        :criterion_type,
-        :start_date,
-        :end_date,
-        :value_as_number,
-        :value_as_string,
-        :value_as_concept_id,
-        :units_source_value,
-        :source_value
-      ]
-
-      attr :values, :options, :arguments, :upstreams
+      extend ConceptQL::Metadatable
+
+      attr_reader :nodifier, :values, :options, :arguments, :upstreams, :op_name, :errors, :warnings
 
       option :label, type: :string
 
-      def initialize(*args)
-        set_values(*args)
-      end
+      @validations = []
+
+      class << self
+        attr_reader :validations, :required_columns
+
+        def register(file, *data_models)
+          data_models = STATIC_OPERATORS.keys if data_models.empty?
+          data_models.each do |dm|
+            op_name = File.basename(file).sub(/\.rb\z/, "").downcase
+            Operators.static_operators[dm][op_name] = self
+          end
+        end
+
+        def query_columns(*tables)
+          define_method(:query_cols) do
+            dm.table_columns(*tables)
+          end
+        end
+
+        def default_query_columns
+          define_method(:query_cols) do
+            dynamic_columns
+          end
+        end
 
-      def values=(*args)
-        set_values(*args)
+        def require_column(column)
+          @required_columns ||= []
+          @required_columns << column
+        end
+
+        validation_meths = <<-END.split.map(&:to_sym)
+          no_upstreams
+          one_upstream
+          at_least_one_upstream
+          at_most_one_upstream
+          no_arguments
+          one_argument
+          at_least_one_argument
+          at_most_one_argument
+          option
+          required_options
+          codes_match
+        END
+
+        validation_meths.each do |type|
+          meth = :"validate_#{type}"
+          define_method(meth) do |*args|
+            validations << [ meth, *args ]
+          end
+        end
+
+        # Declares inherited options that this operator does not support.
+        # They leave the operator's metadata, and a statement that sets one
+        # fails validation.
+        def unsupported_options(*names)
+          names.each { |name| remove_option(name) }
+          validations << [ :validate_unsupported_options, *names ]
+        end
+
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@validations, validations.dup)
+        end
+
+        def new(*)
+          operator = super
+
+          # If operator has a label, replace it with a recall so all references
+          # to it use the same code.
+          if operator.label && !operator.errors
+            operator.scope.add_operator(operator)
+            operator = Operators::Recall.new(operator.nodifier, "recall", operator.label, replaced: true)
+          end
+
+          operator
+        end
       end
 
-      def set_values(*args)
-        @options = args.extract_options!.deep_rekey
-        @upstreams, @arguments = args.partition { |arg| arg.is_a?(Operator) }
+      def initialize(nodifier, op_name, *args)
+        @nodifier = nodifier
+
+        # Under what name was this operator instantiated?
+        # For operators like "vocabulary", this tells the instance which
+        # vocabulary operator to impersonate
+        @op_name = op_name
+        @options = {}
+        @options = @options.merge(ConceptQL::Utils.rekey(args.pop)) while args.last.is_a?(Hash)
+        args.reject! { |arg| arg.nil? || arg == "" }
+        @upstreams, @arguments = args.partition { |arg| arg.is_a?(Array) || arg.is_a?(Operator) }
         @values = args
+
+        scope.nest(self) do
+          create_upstreams
+        end
+      end
+
+      def create_upstreams
+        @upstreams.map! { |stmt| to_op(stmt) }
+      end
+
+      def to_op(stmt)
+        stmt.is_a?(Operator) ? stmt : nodifier.create(*stmt)
+      end
+
+      def operator_name
+        self.class.just_class_name.underscore
+      end
+
+      def required_columns
+        cols = self.class.required_columns || []
+        cols |= [ :uuid ] if options[:uuid]
+        cols
+      end
+
+      def dynamic_columns
+        scope.query_columns
+      end
+
+      def cte_name(name)
+        scope.cte_name(name)
+      end
+
+      def annotate(db, opts = {})
+        return @annotate if defined?(@annotate)
+
+        scope_key = options[:id] || op_name
+        annotation = {}
+        counts = (annotation[:counts] ||= {})
+        metadata = { annotation: annotation }
+
+        if (name = preferred_name)
+          metadata[:name] = name
+        end
+        res = [ op_name, *annotate_values(db, opts) ]
+
+        if upstreams_valid?(db, opts) && scope.valid? && include_counts?(db, opts)
+          scope.with_ctes(self, db)
+               .from_self
+               .select_group(:criterion_domain)
+               .select_append { Sequel.function(:count, 1).as(:rows) }
+               .select_append { count(:person_id).distinct.as(:n) }
+               .each do |h|
+            counts[h.delete(:criterion_domain).to_sym] = h
+          end
+        elsif !errors.empty?
+          annotation[:errors] = errors
+          scope.add_errors(scope_key, errors)
+        end
+
+        domains(db).each do |domain|
+          cur_counts = counts[domain] ||= { rows: 0, n: 0 }
+          scope.add_counts(scope_key, domain, cur_counts)
+        end
+
+        if defined?(@warnings) && !warnings.empty?
+          annotation[:warnings] = warnings
+          scope.add_warnings(scope_key, warnings.dup)
+        end
+
+        if res.last.is_a?(Hash)
+          res.last.merge!(metadata)
+        else
+          res << metadata
+        end
+
+        @annotate = res
+      end
+
+      def code_list(db)
+        upstreams.flat_map { |upstream_op| upstream_op.code_list(db) }
       end
 
-      def scope=(scope)
-        @scope = scope
-        scope.add_operator(self)
+      def inspect
+        "<##{self.class} upstreams=[#{upstreams.map(&:inspect).join(', ')}] arguments=[#{arguments.map(&:inspect).join(', ')}]>"
       end
 
       def evaluate(db)
-        select_it(query(db))
+        q = select_it(query(db))
+        name = cte_name(op_name)
+        opts = {}
+        opts[:materialized] = true if rdbms.supports_materialized?
+        db[name].with(name, q, opts)
+      end
+
+      def pretty_print(pp)
+        pp.object_group self do
+          unless pp_upstreams.empty?
+            pp.breakable
+            pp.text "@upstreams="
+            pp.pp pp_upstreams
+            pp.comma_breakable unless arguments.empty?
+          end
+          unless arguments.empty?
+            pp.breakable if pp_upstreams.empty?
+            pp.text "@arguments="
+            pp.pp arguments
+          end
+        end
+      end
+
+      def pp_upstreams
+        upstreams
+      end
+
+      def all_upstreams
+        upstreams
       end
 
       def sql(db)
         evaluate(db).sql
       end
 
-      def select_it(query, specific_type = nil)
-        specific_type = type if specific_type.nil? && respond_to?(:type)
-        q = query.select(*columns(query, specific_type))
-        if scope && scope.person_ids && upstreams.empty?
-          q = q.where(person_id: scope.person_ids).from_self
+      def select_it(query, opts = {})
+        specific_table = opts[:specific_table]
+        specific_table ||= dm.determine_table(:source_table)
+        specific_table ||= dm.determine_table(:table)
+        specific_table ||= dm.determine_table(:domain)
+
+        dom = begin
+          opts[:domain] || domain
+        rescue StandardError
+          nil
         end
-        q
+
+        opts = {
+          table: specific_table,
+          criterion_domain: dom,
+          query_columns: override_columns,
+          uuid: opts[:uuid] || options[:uuid]
+        }
+
+        cf = column_family || (specific_table ? ConceptQL::Scope::DEFAULT_COLUMN_FAMILY : nil)
+        opts[:replace] ||= {}
+        opts[:replace][:label] = label if label
+        opts[:replace][:column_family] = cf if cf
+
+        query = query.comment(comment) if comments?
+
+        dm.selectify(query, opts)
       end
 
-      def types
-        @types ||= determine_types
+      def comments?
+        ENV["CONCEPTQL_ENABLE_COMMENTS"] == "true"
+      end
+
+      def comment
+        PP.pp(self, "".dup, 10)
+      end
+
+      def override_columns
+        nil
+      end
+
+      def domains(db)
+        @domains ||= determine_domains(db)
+      end
+
+      def tables
+        @tables ||= determine_tables
       end
 
       def stream
         @stream ||= upstreams.first
       end
 
-      def columns(query, local_type = nil)
-        criterion_type = :criterion_type
-        if local_type
-          criterion_type = Sequel.cast_string(local_type.to_s).as(:criterion_type)
-        end
-        columns = [:person_id,
-                    type_id(local_type),
-                    criterion_type]
-        columns += date_columns(query, local_type)
-        columns += value_columns(query, local_type)
+      def columns(table = nil)
+        dm.columns(table: table)
       end
 
       def label
-        options[:label]
+        @label ||= begin
+          options.delete(:label) if options[:label] && options[:label].to_s.strip.empty?
+          options[:label].respond_to?(:strip) ? options[:label].strip : options[:label]
+        end
       end
 
-      private
-      attr :scope
+      def column_family; end
+
+      def valid?(db, opts = {})
+        validate(db, opts)
+        errors.empty?
+      end
+
+      def upstreams_valid?(db, opts = {})
+        valid?(db, opts) && upstreams.all? { |u| u.upstreams_valid?(db, opts) }
+      end
+
+      def scope
+        nodifier.scope
+      end
+
+      def data_model
+        nodifier.data_model
+      end
+
+      def database_type
+        nodifier.database_type
+      end
+
+      def cast_column(column, value = nil)
+        type = Scope::COLUMN_TYPES.fetch(column)
+        case type
+        when String, :String
+          Sequel.cast_string(value).as(column)
+        when Date, :Date
+          Sequel.cast(value, type).as(column)
+        when Float, :Bigint, :Float
+          Sequel.cast_numeric(value, type).as(column)
+        else
+          raise "Unexpected type: '#{type.inspect}' for column: '#{column}'"
+        end
+      end
+
+      def omopv4_plus?
+        data_model == :omopv4_plus
+      end
+
+      def gdm?
+        %i[gdm gdm_wide].include?(data_model)
+      end
 
-      def criterion_id
-        :criterion_id
+      def wide?
+        %i[gdm_wide].include?(data_model)
       end
 
-      def type_id(type = nil)
-        return :criterion_id if type.nil?
-        type = :person if type == :death
-        Sequel.expr(make_type_id(type)).as(:criterion_id)
+      def dm
+        @dm ||= DataModel.for(self, nodifier)
       end
 
-      def make_type_id(type)
-        (type.to_s + '_id').to_sym
+      def rdbms
+        dm.rdbms
+      end
+
+      def query_cols
+        raise NotImplementedError, self
+      end
+
+      def events_per_patient
+        if upstreams.first
+          upstreams.first.events_per_patient
+        else
+          :multiple
+        end
+      end
+
+      def vocabularies
+        if respond_to?(:vocab_entry)
+          [ vocab_entry.omopv5_vocabulary_id ]
+        elsif upstreams.any?
+          upstreams.flat_map(&:vocabularies).uniq
+        else
+          []
+        end
+      end
+
+      def multiple_vocabularies
+        if upstreams.any?
+          upstreams.first.multiple_vocabularies
+        else
+          false
+        end
+      end
+
+      def accept(visitor, opts = {})
+        visitor.visit(self)
+        upstreams.each { |u| u.accept(visitor, opts) }
+      end
+
+      private
+
+      def annotate_values(db, opts)
+        (upstreams.map { |op| op.annotate(db, opts) } + arguments).push(options)
       end
 
       def make_table_name(table)
-        "#{table}___tab".to_sym
-      end
-
-      def value_columns(query, type)
-        [
-          numeric_value(query),
-          string_value(query),
-          concept_id_value(query),
-          units_source_value(query),
-          source_value(query, type)
-        ]
-      end
-
-      def numeric_value(query)
-        return :value_as_number if query.columns.include?(:value_as_number)
-        Sequel.cast_numeric(nil, Float).as(:value_as_number)
-      end
-
-      def string_value(query)
-        return :value_as_string if query.columns.include?(:value_as_string)
-        Sequel.cast_string(nil).as(:value_as_string)
-      end
-
-      def concept_id_value(query)
-        return :value_as_concept_id if query.columns.include?(:value_as_concept_id)
-        Sequel.cast_numeric(nil).as(:value_as_concept_id)
-      end
-
-      def units_source_value(query)
-        return :units_source_value if query.columns.include?(:units_source_value)
-        Sequel.cast_string(nil).as(:units_source_value)
-      end
-
-      def source_value(query, type)
-        return :source_value if query.columns.include?(:source_value)
-        Sequel.cast_string(source_value_column(query, type)).as(:source_value)
-      end
-
-      def date_columns(query, type = nil)
-        return [:start_date, :end_date] if (query.columns.include?(:start_date) && query.columns.include?(:end_date))
-        return [:start_date, :end_date] unless type
-        sd = start_date_column(query, type)
-        sd = Sequel.expr(sd).cast(:date).as(:start_date) unless sd == :start_date
-        ed = end_date_column(query, type)
-        ed = Sequel.function(:coalesce, Sequel.expr(ed).cast(:date), Sequel.expr(start_date_column(query, type))).as(:end_date) unless ed == :end_date
-        [sd, ed]
-      end
-
-      def start_date_column(query, type)
-        {
-          condition_occurrence: :condition_start_date,
-          death: :death_date,
-          drug_exposure: :drug_exposure_start_date,
-          drug_cost: nil,
-          payer_plan_period: :payer_plan_period_start_date,
-          person: person_date_of_birth(query),
-          procedure_occurrence: :procedure_date,
-          procedure_cost: nil,
-          observation: :observation_date,
-          observation_period: :observation_period_start_date,
-          visit_occurrence: :visit_start_date
-        }[type]
-      end
-
-      def end_date_column(query, type)
-        {
-          condition_occurrence: :condition_end_date,
-          death: :death_date,
-          drug_exposure: :drug_exposure_end_date,
-          drug_cost: nil,
-          payer_plan_period: :payer_plan_period_end_date,
-          person: person_date_of_birth(query),
-          procedure_occurrence: :procedure_date,
-          procedure_cost: nil,
-          observation: :observation_date,
-          observation_period: :observation_period_end_date,
-          visit_occurrence: :visit_end_date
-        }[type]
-      end
-
-      def source_value_column(query, type)
-        {
-          condition_occurrence: :condition_source_value,
-          death: :cause_of_death_source_value,
-          drug_exposure: :drug_source_value,
-          drug_cost: nil,
-          payer_plan_period: :payer_plan_period_source_value,
-          person: :person_source_value,
-          procedure_occurrence: :procedure_source_value,
-          procedure_cost: nil,
-          observation: :observation_source_value,
-          observation_period: nil,
-          visit_occurrence: :place_of_service_source_value
-        }[type]
-      end
-
-      def person_date_of_birth(query)
-        assemble_date(query, :year_of_birth, :month_of_birth, :day_of_birth)
-      end
-
-      def assemble_date(query, *symbols)
-        strings = symbols.map do |symbol|
-          Sequel.cast_string(Sequel.function(:coalesce, Sequel.cast_string(symbol), Sequel.cast_string('01')))
-        end
-        strings = strings.zip(['-'] * (symbols.length - 1)).flatten.compact
-        concatted_strings = Sequel.join(strings)
-        case query.db.database_type
-        when :oracle
-          Sequel.function(:to_date, concatted_strings, 'YYYY-MM-DD')
-        when :mssql
-          Sequel.lit('CONVERT(DATETIME, ?)', concatted_strings)
+        Sequel.as(table, :tab)
+      end
+
+      def query_columns(query)
+        dm.query_columns(query)
+      end
+
+      def additional_columns(query, table)
+        special_columns = {
+          code_provenance_type: proc { code_provenance_type(query, table) },
+          file_provenance_type: proc { file_provenance_type(query, table) },
+          provider_id: proc { provider_id(query, table) },
+          visit_source_concept_id: proc { dm.place_of_service_concept_id(query, table) }
+        }
+
+        additional_cols = special_columns.each_with_object([]) do |(column, proc_obj), columns|
+          columns << proc_obj.call if dynamic_columns.include?(column)
+        end
+
+        standard_columns = dynamic_columns - Scope::DEFAULT_COLUMNS.keys
+        standard_columns -= special_columns.keys
+
+        standard_columns.each do |column|
+          additional_cols << if query_columns(query).include?(column)
+                               column
+          else
+                               cast_column(column)
+          end
+        end
+
+        additional_cols
+      end
+
+      def source_value(query, table)
+        return :source_value if query_columns(query).include?(:source_value)
+
+        cast_column(:source_value, dm.source_value_column(query, table))
+      end
+
+      def source_vocabulary_id(query, table)
+        return :source_vocabulary_id if query_columns(query).include?(:source_vocabulary_id)
+
+        cast_column(:source_vocabulary_id, dm.source_vocabulary_id(query, table))
+      end
+
+      def code_provenance_type(query, table)
+        return :code_provenance_type if query_columns(query).include?(:code_provenance_type)
+
+        cast_column(:code_provenance_type, dm.provenance_type_column(query, table))
+      end
+
+      def file_provenance_type(query, table)
+        return :file_provenance_type if query_columns(query).include?(:file_provenance_type)
+
+        cast_column(:file_provenance_type, dm.provenance_type_column(query, table))
+      end
+
+      def provider_id(query, table)
+        return :provider_id if query_columns(query).include?(:provider_id)
+
+        cast_column(:provider_id, dm.provider_id_column(query, table))
+      end
+
+      def determine_tables
+        if upstreams.empty?
+          if respond_to?(:table)
+            [ table ]
+          else
+            [ :invalid ]
+          end
         else
-          Sequel.cast(concatted_strings, Date)
+          tables = upstreams.compact.map(&:tables).flatten.uniq
+          tables.empty? ? [ :invalid ] : tables
         end
       end
 
-      def determine_types
+      def determine_domains(db)
         if upstreams.empty?
-          if respond_to?(:type)
-            [type]
+          if respond_to?(:domain)
+            [ domain ]
           else
-            raise "Operator doesn't seem to specify any type"
+            [ :invalid ]
           end
         else
-          upstreams.map(&:types).flatten.uniq
+          doms = upstreams.compact.flat_map { |up| up.domains(db) }.uniq
+          doms.empty? ? [ :invalid ] : doms
+        end
+      end
+
+      # Validation Related
+
+      def upstream_operator_names
+        @upstreams.map(&:op_name)
+      end
+
+      def validate(db, opts = {})
+        return if @_validated
+
+        @errors = [] unless defined?(@errors)
+        @warnings = [] unless defined?(@warnings)
+
+        add_error("invalid label") if label && !label.is_a?(String)
+        self.class.validations.each do |args|
+          send(*args)
+        end
+
+        additional_validation(db, opts)
+
+        @_validated = true
+      end
+
+      def validate_no_upstreams
+        add_error("has upstreams", upstream_operator_names) unless @upstreams.empty?
+      end
+
+      def validate_one_upstream
+        validate_at_least_one_upstream
+        validate_at_most_one_upstream
+      end
+
+      def validate_at_most_one_upstream
+        add_error("has multiple upstreams", upstream_operator_names) if @upstreams.length > 1
+      end
+
+      def validate_at_least_one_upstream
+        add_error("has no upstream") if @upstreams.empty?
+      end
+
+      def validate_no_arguments
+        add_error("has arguments", @arguments) unless @arguments.empty?
+      end
+
+      def validate_one_argument
+        validate_at_least_one_argument
+        validate_at_most_one_argument
+      end
+
+      def validate_at_most_one_argument
+        add_error("has multiple arguments", @arguments) if @arguments.length > 1
+      end
+
+      def validate_at_least_one_argument
+        add_error("has no arguments") if @arguments.empty?
+      end
+
+      def validate_option(format, *opts)
+        opts.each do |opt|
+          next unless options.key?(opt)
+
+          add_error("wrong option format", opt.to_s, options[opt]) unless format === options[opt]
+        end
+      end
+
+      def validate_required_options(*opts)
+        opts.each do |opt|
+          add_error("required option not present", opt.to_s) unless options.key?(opt)
+        end
+      end
+
+      # A blank value counts as not set, as in Occurrence#at_least_option
+      def validate_unsupported_options(*opts)
+        opts.each do |opt|
+          next if options[opt].to_s.strip.empty?
+
+          add_error("unsupported option", opt.to_s, options[opt])
+        end
+      end
+
+      def bad_arguments
+        return [] unless code_regexp
+        # Vocabulary#select_all? is private, so include private methods.
+        return [] if respond_to?(:select_all?, true) && select_all?
+
+        @bad_arguments ||= arguments.reject do |arg|
+          code_regexp === arg
+        end
+      end
+
+      # Only Vocabulary operators carry a code format (Vocabulary#code_regexp)
+      def code_regexp
+        nil
+      end
+
+      def validate_codes_match
+        return if bad_arguments.empty?
+
+        add_warning("improperly formatted code", *bad_arguments)
+      end
+
+      def additional_validation(_db, _opts = {})
+        # Do nothing by default
+      end
+
+      def add_warnings?(db, opts = {})
+        @errors.empty? && !no_db?(db, opts)
+      end
+
+      def add_error(*args)
+        errors << args
+      end
+
+      def add_warning(*args)
+        warnings << args
+      end
+
+      def include_counts?(db, opts)
+        !(no_db?(db, opts) || opts[:skip_counts])
+      end
+
+      def skip_db?(opts)
+        opts[:skip_db]
+      end
+
+      def no_db?(db, opts = {})
+        no_db = db.nil? || db.adapter_scheme == :mock || skip_db?(opts)
+        no_db ||= table_is_missing?(db)
+        no_db
+      end
+
+      def table_is_missing?(_db)
+        false
+      end
+
+      def preferred_name
+        self.class.pref_name
+      end
+
+      def semi_or_inner_join(ds, table, *exprs)
+        ds = Sequel[ds] if ds.is_a?(Symbol)
+        table = Sequel[table] if table.is_a?(Symbol)
+        expr = exprs.inject(&:&)
+        if use_inner_join?
+          ds.join(table.as(:r), expr)
+            .select(*query_cols.map { |c| Sequel[:l][c] })
+        else
+          rdbms.semi_join(ds, table, *exprs)
+        end
+      end
+
+      def matching_columns
+        if scope.opts.dig(:window_opts, :window_table)
+          %i[person_id window_id]
+        else
+          [ :person_id ]
+        end
+      end
+
+      def use_inner_join?
+        options[:inner_join]
+      end
+
+      # The names in an option that lists columns (join_columns, only_columns
+      # and so on), as symbols. A JSON statement gives them as strings, and
+      # Sequel renders a String as a string literal, not as a column. Call it
+      # only for options that name columns: other options are free-form.
+      #
+      # The value is an Array or a comma-separated String, such as
+      # "uuid, start_date". A blank String is an unset option, so it gives
+      # nil.
+      def column_names_option(name)
+        value = options[name]
+        if value.is_a?(String)
+          value = value.split(",").map(&:strip).reject(&:empty?)
+          return nil if value.empty?
         end
+        value&.map(&:to_sym)
       end
     end
   end