conceptql 0.2.0 → 0.3.0

data/lib/conceptql/operators/numeric.rb

@@ -12,12 +12,14 @@ module ConceptQL
     # - Either a number value or a symbol representing a column name
     # - An optional stream
     class Numeric < PassThru
+      register __FILE__
+
       desc <<-EOF
-Represents a operator that will either:
-- create a value_as_number value for every person in the database
-- change the value_as_number value for every every result passed in
-  - either to a number
-  - or a value from a column in the origin row
+Represents an operator that will either:
+- Create a value_as_number value for every person in the database
+- Change the value_as_number value for every result passed in
+  - Either to a number
+  - Or a value from a column in the origin row
 
 Accepts two params:
 - Either a number value or a symbol representing a column name
@@ -25,29 +27,46 @@ Accepts two params:
       EOF
       argument :value, type: :float
       allows_one_upstream
+      validate_at_most_one_upstream
+      validate_one_argument
+      default_query_columns
+      require_column :value_as_number
+
+      def query_cols
+        dynamic_columns - [:value_as_number] + [:value_as_number]
+      end
 
       def query(db)
         stream.nil? ? as_criterion(db) : with_kids(db)
       end
 
-      def types
+      def domains(db)
         stream.nil? ? [:person] : super
       end
 
       private
       def with_kids(db)
         db.from(stream.evaluate(db))
-          .select(*(COLUMNS - [:value_as_number]))
-          .select_append(Sequel.lit('?', arguments.first).cast(Float).as(:value_as_number))
+          .select(*(dynamic_columns - [:value_as_number]))
+          .select_append(first_argument.cast(Float).as(:value_as_number))
           .from_self
       end
 
       def as_criterion(db)
-        db.from(select_it(db.from(:person), :person))
-          .select(*(COLUMNS - [:value_as_number]))
-          .select_append(Sequel.lit('?', arguments.first).cast(Float).as(:value_as_number))
+        db.from(select_it(db.from(:person).clone(:force_columns=>table_columns(:person)), :person))
+          .select(*(dynamic_columns - [:value_as_number]))
+          .select_append(first_argument.cast(Float).as(:value_as_number))
           .from_self
       end
+
+      def first_argument
+        case arguments.first
+        when String
+          Sequel.identifier(arguments.first)
+        else
+          Sequel.expr(arguments.first)
+        end
+      end
     end
   end
 end

data/lib/conceptql/operators/observation_by_enttype.rb

@@ -1,11 +1,14 @@
 require_relative 'source_vocabulary_operator'
+require_relative '../behaviors/labish'
 
 module ConceptQL
   module Operators
     class ObservationByEnttype < SourceVocabularyOperator
-      desc 'Searches the observation table for all observations with matching Enttype'
+      register __FILE__
+
       argument :enttypes, type: :codelist, vocab_id: [206, 207]
-      predominant_types :observation
+      predominant_domains :observation
+      include ConceptQL::Labish
 
       def table
         :observation

data/lib/conceptql/operators/observation_period.rb

@@ -3,11 +3,13 @@ require_relative 'casting_operator'
 module ConceptQL
   module Operators
     class ObservationPeriod < CastingOperator
+      register __FILE__
+
       desc 'Generates all observation_period records, or, if fed a stream, fetches all observation_period records for the people represented in the incoming result set.'
-      types :observation_period
+      domains :observation_period
       allows_one_upstream
 
-      def my_type
+      def my_domain
         :observation_period
       end
 
@@ -16,12 +18,12 @@ module ConceptQL
       end
 
       def these_point_at_me
-        # I could list ALL the types we use, but the default behavior of casting,
+        # I could list ALL the domains we use, but the default behavior of casting,
         # when there is no explicit casting defined, is to convert everything to
         # person IDs
         #
         # So by defining no known castable relationships in this operator, all
-        # types will be converted to person
+        # domains will be converted to person
         []
       end
     end

data/lib/conceptql/operators/occurrence.rb

@@ -22,32 +22,39 @@ module ConceptQL
     # If we ask for the second occurrence of something and a person has only one
     # occurrence, this operator returns nothing for that person
     class Occurrence < Operator
+      register __FILE__
+
       preferred_name 'Nth Occurrence'
+
       desc <<-EOF
-Groups all results by person, then orders by start_date, then finds the nth occurrence.
-nth occurrence can be positive or negative.
+Groups all results by person, then orders by start_date and finds the nth occurrence (can be positive or negative).
 1 => first
 2 => second
 ...
 -1 => last
 -2 => second-to-last
 
-If two rows have the same start_date, the order of their ranking
-is arbitrary
+If two results have the same start_date, their relative order
+is arbitrary.
 
 If we ask for the second occurrence of something and a person has only one
-occurrence, this operator returns nothing for that person
+occurrence, this operator returns nothing for that person.
       EOF
+
       argument :occurrence, type: :integer
+      category "Filter Single Stream"
+      basic_type :temporal
       allows_one_upstream
-      category %w(Temporal Occurrence)
+      validate_at_least_one_upstream
+      option :unique, type: :boolean, label: 'Unique Source Values Only'
+
+      def query_cols
+        dynamic_columns + [:rn]
+      end
 
       def query(db)
         db[:occurrences]
-          .with(:occurrences,
-            stream.evaluate(db)
-              .from_self
-              .select_append { |o| o.row_number(:over, partition: :person_id, order: ordered_columns){}.as(:rn) })
+          .with(:occurrences, occurrences(db))
           .where(rn: occurrence.abs)
       end
 
@@ -55,14 +62,43 @@ occurrence, this operator returns nothing for that person
         @occurrence ||= arguments.first
       end
 
+      def occurrences(db)
+        all_or_uniquified_results(db)
+          .from_self
+          .select_append { |o| o.row_number(:over, partition: :person_id, order: ordered_columns){}.as(:rn) }
+      end
+
       private
+
+      def validate(db, opts = {})
+        super
+        if self.class == Occurrence
+          validate_one_argument
+        else
+          validate_no_arguments
+        end
+      end
+
       def asc_or_desc
         occurrence < 0 ? :desc : :asc
       end
 
       def ordered_columns
         ordered_columns = [Sequel.send(asc_or_desc, :start_date)]
-        ordered_columns += [:criterion_type, :criterion_id]
+        ordered_columns += [:criterion_id]
+      end
+
+      def uniquify_partition_columns
+        dynamic_columns - [:criterion_id, :start_date, :end_date]
+      end
+
+      def all_or_uniquified_results(db)
+        return stream.evaluate(db) unless options[:unique]
+        stream.evaluate(db)
+          .from_self
+          .select_append { |o| o.row_number(:over, partition: uniquify_partition_columns, order: ordered_columns){}.as(:unique_rn) }
+          .from_self
+          .where(unique_rn: 1)
       end
     end
   end

data/lib/conceptql/operators/one_in_two_out.rb

@@ -1,60 +1,123 @@
 require_relative 'operator'
 require_relative 'visit_occurrence'
+require_relative '../date_adjuster'
+require_relative '../behaviors/provenanceable'
+require 'facets/kernel/blank'
 
 module ConceptQL
   module Operators
     class OneInTwoOut < Operator
+      register __FILE__
+
+      include ConceptQL::Provenanceable
+
       desc <<-EOF
-Represents a common pattern in research algorithms: searching for a condition
-that appears either two times in an outpatient setting with a 30-day gap or once
-in an inpatient setting
+Represents a common pattern in research algorithms: searching for an event
+that appears either once in an inpatient setting or
+twice in an outpatient setting with a 30-day gap.
       EOF
       allows_one_upstream
-      category %w(Temporal Relative)
+      validate_one_upstream
+      validate_no_arguments
+      category "Filter Single Stream"
+      basic_type :temporal
 
-      def types
-        [:visit_occurrence]
-      end
+      option :inpatient_length_of_stay, type: :integer, min: 0, default: 0, desc: 'Minimum length of inpatient stay (in days) required for inpatient event to be valid', label: 'Inpatient Length of Stay (Days)'
+      option :inpatient_return_date, type: :string, options: ['Admit Date', 'Discharge Date'], default: 'Discharge Date', desc: 'Which date to pass downstream in both the start_date and end_date fields'
+      option :outpatient_minimum_gap, type: :string, default: '30d', desc: 'Minimum number of days between outpatient events for the event to be valid'
+      option :outpatient_maximum_gap, type: :string, desc: 'Maximum number of days between outpatient events for the event to be valid'
+      option :outpatient_event_to_return, type: :string, options: ['Initial Event', 'Confirming Event'], default: 'Initial Event', desc: 'Which event to pass downstream'
+
+      validate_option DateAdjuster::VALID_INPUT, :outpatient_minimum_gap, :outpatient_maximum_gap
+
+      default_query_columns
+
+      require_column :provenance_type
+
+      attr_reader :db
 
       def query(db)
-        inpatient = select_it(visit_query(db).where(place_of_service_concept_id: 8717), :visit_occurrence).from_self
-        outpatient = select_it(visit_query(db).exclude(place_of_service_concept_id: 8717), :visit_occurrence).from_self
-
-        gap = options[:gap] || 30
-        valid_outpatient_people = outpatient
-          .group_by(:person_id)
-          .select(:person_id)
-          .select_append(Sequel.expr(Sequel.function(:max, :start_date) - Sequel.function(:min, :end_date)).as(:date_diff))
+        @db = db
+        first_valid_event.from_self
+      end
+
+      private
+
+      def condition_events
+        db[stream.evaluate(db)]
+          .where(criterion_domain: 'condition_occurrence')
+          .exclude(provenance_type: nil)
           .from_self
-            .where{ date_diff >= gap}
+      end
 
-        relevant_outpatient = outpatient.where(person_id: valid_outpatient_people.select(:person_id))
-        earliest(db, inpatient.union(relevant_outpatient))
+      def all_inpatient_events
+        condition_events
+          .where(provenance_type: to_concept_id(:inpatient))
+          .from_self
       end
 
-      private
-      def visit_query(db)
-        VisitOccurrence.new(FakeOperator.new(stream.evaluate(db).from_self, stream.types)).query(db)
+      def valid_inpatient_events
+        q = all_inpatient_events
+        unless options[:inpatient_length_of_stay].nil? || options[:inpatient_length_of_stay].to_i.zero?
+          q = q.where{ |o| Sequel.date_sub(o.end_date, o.start_date) > options[:inpatient_length_of_stay].to_i }
+        end
+
+        if options[:inpatient_return_date] != 'Admit Date'
+          q = q.select(*(query_cols - [:start_date])).select_append(:end_date___start_date)
+        end
+
+        q.from_self.select(*dynamic_columns).from_self
       end
 
-      def earliest(db, query)
-        db[:earliest]
-          .with(:earliest,
-            query.select_append { |o| o.row_number(:over, partition: :person_id, order: [Sequel.asc(:start_date), :criterion_type, :criterion_id]){}.as(:rn) })
-          .where(rn: 1)
+      def outpatient_events
+        condition_events
+          .where(provenance_type: to_concept_id(:claim) + to_concept_id(:outpatient))
           .from_self
       end
 
-      class FakeOperator < Operator
-        attr :types
-        def initialize(query, types)
-          @query = query
-          @types = types
+      def valid_outpatient_events
+
+        min_gap = options[:outpatient_minimum_gap] || "30d"
+
+        max_gap = options[:outpatient_maximum_gap]
+
+        q = outpatient_events.from_self(alias: :initial)
+              .join(outpatient_events.as(:confirm), initial__person_id: :confirm__person_id)
+              .exclude(initial__criterion_id: :confirm__criterion_id)
+
+        # In order to avoid many more comparisons of initial to confirm events, we now
+        # filter the join by having only confirm events that come on or after initial events
+        #
+        # This ensures that initial events represent initial events and confirm events
+        # represent confirming events
+        q = q.exclude{confirm__start_date < initial__start_date}
+
+        if min_gap.present?
+          q = q.where { confirm__start_date >= DateAdjuster.new(min_gap).adjust(:initial__start_date) }
+        end
+
+        if max_gap.present?
+          q = q.where { confirm__start_date <= DateAdjuster.new(max_gap).adjust(:initial__start_date) }
         end
 
-        def query(db)
-          @query
+        if options[:outpatient_event_to_return] != 'Initial Event'
+          q = q.select_all(:confirm)
+        else
+          q = q.select_all(:initial)
         end
+
+        q.from_self
+      end
+
+      def all_valid_events
+        valid_inpatient_events.union(valid_outpatient_events, all: true)
+      end
+
+      def first_valid_event
+        all_valid_events
+          .select_append { |o| o.row_number(:over, partition: :person_id, order: [ :start_date, :criterion_id ]){}.as(:rn) }
+          .from_self
+          .where(rn: 1)
       end
     end
   end

data/lib/conceptql/operators/operator.rb

@@ -3,46 +3,246 @@ require_relative '../behaviors/metadatable'
 require 'facets/array/extract_options'
 require 'facets/hash/deep_rekey'
 require 'forwardable'
+require_relative '../query_modifiers/pos_query_modifier'
+require_relative '../query_modifiers/drug_query_modifier'
 
 module ConceptQL
   module Operators
+    OPERATORS = {:omopv4=>{}, :omopv4_plus=>{}}.freeze
+
+    TABLE_VOCABULARY_ID_COLUMN = {
+      :condition_occurrence=> :condition_source_vocabulary_id,
+      :death=> :cause_of_death_source_vocabulary_id,
+      :drug_exposure=> :drug_source_vocabulary_id,
+      :observation=> :observation_source_vocabulary_id,
+      :procedure_occurrence=> :procedure_source_vocabulary_id,
+      :provider=> :specialty_source_vocabulary_id,
+      :visit_occurrence=> :place_of_service_source_vocabulary_id
+    }.freeze.each_value(&:freeze)
+
+    TABLE_SOURCE_VALUE_COLUMN = {
+      :condition_occurrence=> :condition_source_value,
+      :death=> :cause_of_death_source_value,
+      :drug_exposure=> :drug_source_value,
+      :observation=> :observation_source_value,
+      :procedure_occurrence=> :procedure_source_value,
+      :provider=> :provider_source_value,
+      :visit_occurrence=> :place_of_service_source_value
+    }.freeze.each_value(&:freeze)
+
+    TABLE_COLUMNS = {
+      :care_site=>[:care_site_id, :location_id, :organization_id, :place_of_service_concept_id, :care_site_source_value, :place_of_service_source_value],
+      :cohort=>[:cohort_id, :cohort_concept_id, :cohort_start_date, :cohort_end_date, :subject_id, :stop_reason],
+      :concept=>[:concept_id, :concept_name, :concept_level, :concept_class, :vocabulary_id, :concept_code, :valid_start_date, :valid_end_date, :invalid_reason],
+      :concept_ancestor=>[:ancestor_concept_id, :descendant_concept_id, :min_levels_of_separation, :max_levels_of_separation],
+      :concept_relationship=>[:concept_id_1, :concept_id_2, :relationship_id, :valid_start_date, :valid_end_date, :invalid_reason],
+      :concept_synonym=>[:concept_synonym_id, :concept_id, :concept_synonym_name],
+      :condition_era=>[:condition_era_id, :person_id, :condition_concept_id, :condition_era_start_date, :condition_era_end_date, :condition_type_concept_id, :condition_occurrence_count],
+      :condition_occurrence=>[:condition_occurrence_id, :person_id, :condition_concept_id, :condition_start_date, :condition_end_date, :condition_type_concept_id, :stop_reason, :associated_provider_id, :visit_occurrence_id, :condition_source_value],
+      :death=>[:person_id, :death_date, :death_type_concept_id, :cause_of_death_concept_id, :cause_of_death_source_value],
+      :drug_approval=>[:ingredient_concept_id, :approval_date, :approved_by],
+      :drug_cost=>[:drug_cost_id, :drug_exposure_id, :paid_copay, :paid_coinsurance, :paid_toward_deductible, :paid_by_payer, :paid_by_coordination_benefits, :total_out_of_pocket, :total_paid, :ingredient_cost, :dispensing_fee, :average_wholesale_price, :payer_plan_period_id],
+      :drug_era=>[:drug_era_id, :person_id, :drug_concept_id, :drug_era_start_date, :drug_era_end_date, :drug_type_concept_id, :drug_exposure_count],
+      :drug_exposure=>[:drug_exposure_id, :person_id, :drug_concept_id, :drug_exposure_start_date, :drug_exposure_end_date, :drug_type_concept_id, :stop_reason, :refills, :quantity, :days_supply, :sig, :prescribing_provider_id, :visit_occurrence_id, :relevant_condition_concept_id, :drug_source_value],
+      :drug_strength=>[:drug_concept_id, :ingredient_concept_id, :amount_value, :amount_unit, :concentration_value, :concentration_enum_unit, :concentration_denom_unit, :valid_start_date, :valid_end_date, :invalid_reason],
+      :location=>[:location_id, :address_1, :address_2, :city, :state, :zip, :county, :location_source_value],
+      :observation=>[:observation_id, :person_id, :observation_concept_id, :observation_date, :observation_time, :value_as_number, :value_as_string, :value_as_concept_id, :unit_concept_id, :range_low, :range_high, :observation_type_concept_id, :associated_provider_id, :visit_occurrence_id, :relevant_condition_concept_id, :observation_source_value, :unit_source_value],
+      :observation_period=>[:observation_period_id, :person_id, :observation_period_start_date, :observation_period_end_date, :prev_ds_period_end_date],
+      :organization=>[:organization_id, :place_of_service_concept_id, :location_id, :organization_source_value, :place_of_service_source_value],
+      :payer_plan_period=>[:payer_plan_period_id, :person_id, :payer_plan_period_start_date, :payer_plan_period_end_date, :payer_source_value, :plan_source_value, :family_source_value, :prev_ds_period_end_date],
+      :person=>[:person_id, :gender_concept_id, :year_of_birth, :month_of_birth, :day_of_birth, :race_concept_id, :ethnicity_concept_id, :location_id, :provider_id, :care_site_id, :person_source_value, :gender_source_value, :race_source_value, :ethnicity_source_value],
+      :procedure_cost=>[:procedure_cost_id, :procedure_occurrence_id, :paid_copay, :paid_coinsurance, :paid_toward_deductible, :paid_by_payer, :paid_by_coordination_benefits, :total_out_of_pocket, :total_paid, :disease_class_concept_id, :revenue_code_concept_id, :payer_plan_period_id, :disease_class_source_value, :revenue_code_source_value],
+      :procedure_occurrence=>[:procedure_occurrence_id, :person_id, :procedure_concept_id, :procedure_date, :procedure_type_concept_id, :associated_provider_id, :visit_occurrence_id, :relevant_condition_concept_id, :procedure_source_value],
+      :provider=>[:provider_id, :npi, :dea, :specialty_concept_id, :care_site_id, :provider_source_value, :specialty_source_value],
+      :relationship=>[:relationship_id, :relationship_name, :is_hierarchical, :defines_ancestry, :reverse_relationship],
+      :source_to_concept_map=>[:source_code, :source_vocabulary_id, :source_code_description, :target_concept_id, :target_vocabulary_id, :mapping_type, :primary_map, :valid_start_date, :valid_end_date, :invalid_reason],
+      :visit_occurrence=>[:visit_occurrence_id, :person_id, :visit_start_date, :visit_end_date, :place_of_service_concept_id, :care_site_id, :place_of_service_source_value],
+      :vocabulary=>[:vocabulary_id, :vocabulary_name],
+
+      # Modifier Tables
+      :drug_appendix=>[:drug_name, :drug_amount, :drug_amount_units]
+    }.freeze.each_value(&:freeze)
+
+    def self.operators
+      OPERATORS
+    end
+
     class Operator
       extend Forwardable
-      extend Metadatable
-      COLUMNS = [
-        :person_id,
-        :criterion_id,
-        :criterion_type,
-        :start_date,
-        :end_date,
-        :value_as_number,
-        :value_as_string,
-        :value_as_concept_id,
-        :units_source_value,
-        :source_value
-      ]
-
-      attr :values, :options, :arguments, :upstreams
+      extend ConceptQL::Metadatable
+
+      attr :nodifier, :values, :options, :arguments, :upstreams
 
       option :label, type: :string
 
-      def initialize(*args)
-        set_values(*args)
-      end
+      @validations = []
 
-      def values=(*args)
-        set_values(*args)
+      class << self
+        attr :validations, :codes_regexp, :required_columns
+
+        def register(file, *data_models)
+          data_models = OPERATORS.keys if data_models.empty?
+          data_models.each do |dm|
+            OPERATORS[dm][File.basename(file).sub(/\.rb\z/, '')] = self
+          end
+        end
+
+        def query_columns(*tables)
+          define_method(:query_cols) do
+            table_columns(*tables)
+          end
+        end
+
+        def default_query_columns
+          define_method(:query_cols) do
+            dynamic_columns
+          end
+        end
+
+        def require_column(column)
+          @required_columns ||= []
+          @required_columns << column
+        end
+
+        validation_meths = (<<-END).split.map(&:to_sym)
+          no_upstreams
+          one_upstream
+          at_least_one_upstream
+          at_most_one_upstream
+          no_arguments
+          one_argument
+          at_least_one_argument
+          at_most_one_argument
+          option
+          required_options
+          codes_match
+        END
+
+        validation_meths.each do |type|
+          meth = :"validate_#{type}"
+          define_method(meth) do |*args|
+            validations << [meth, *args]
+          end
+        end
+
+        def codes_should_match(format)
+          @codes_regexp = format
+          validate_codes_match
+        end
+
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@validations, validations.dup)
+          subclass.instance_variable_set(:@codes_regexp, codes_regexp.dup) if codes_regexp
+        end
+
+        def new(*)
+          operator = super
+
+          # If operator has a label, replace it with a recall so all references
+          # to it use the same code.
+          if operator.label && !operator.errors
+            operator.scope.add_operator(operator)
+            operator = Operators::Recall.new(operator.nodifier, operator.label, replaced: true)
+          end
+
+          operator
+        end
       end
 
-      def set_values(*args)
-        @options = args.extract_options!.deep_rekey
-        @upstreams, @arguments = args.partition { |arg| arg.is_a?(Operator) }
+      def initialize(nodifier, *args)
+        @nodifier = nodifier
+        @options = {}
+        while args.last.is_a?(Hash)
+          @options = @options.merge(args.extract_options!.deep_rekey)
+        end
+        args.reject!{|arg| arg.nil? || arg == ''}
+        @upstreams, @arguments = args.partition { |arg| arg.is_a?(Array) || arg.is_a?(Operator) }
         @values = args
+
+        scope.nest(self) do
+          create_upstreams
+        end
+      end
+
+      def create_upstreams
+        @upstreams.map!{|stmt| to_op(stmt)}
+      end
+
+      def to_op(stmt)
+        stmt.is_a?(Operator) ? stmt : nodifier.create(*stmt)
+      end
+
+      def operator_name
+        self.class.just_class_name.underscore
+      end
+
+      def required_columns
+        self.class.required_columns
+      end
+
+      def dynamic_columns
+        scope.query_columns
+      end
+
+      def annotate(db, opts = {})
+        return @annotation if defined?(@annotation)
+
+        scope_key = options[:id]||self.class.just_class_name.underscore
+        annotation = {}
+        counts = (annotation[:counts] ||= {})
+        metadata = {:annotation=>annotation}
+        if name = self.class.preferred_name
+          metadata[:name] = name
+        end
+        res = [operator_name, *annotate_values(db, opts)]
+
+        if upstreams_valid?(db, opts) && scope.valid? && include_counts?(db, opts)
+          scope.with_ctes(evaluate(db), db)
+            .from_self
+            .select_group(:criterion_domain)
+            .select_append{count{}.*.as(:rows)}
+            .select_append{count(:person_id).distinct.as(:n)}
+            .each do |h|
+              counts[h.delete(:criterion_domain).to_sym] = h
+            end
+        elsif !errors.empty?
+          annotation[:errors] = errors
+          scope.add_errors(scope_key, errors)
+        end
+        scope.add_operators(self)
+        domains(db).each do |domain|
+          cur_counts = counts[domain] ||= {:rows=>0, :n=>0}
+          scope.add_counts(scope_key, domain, cur_counts)
+        end
+
+        if defined?(@warnings) && !warnings.empty?
+          annotation[:warnings] = warnings
+          scope.add_warnings(scope_key, warnings)
+        end
+
+        if res.last.is_a?(Hash)
+          res.last.merge!(metadata)
+        else
+          res << metadata
+        end
+
+        @annotation = res
       end
 
-      def scope=(scope)
-        @scope = scope
-        scope.add_operator(self)
+      def code_list(db)
+        code_lists = @upstreams.map do | upstream_op |
+          upstream_op.code_list(db)
+        end
+        code_lists.flatten(1)
+      end
+
+      def dup_values(args)
+        self.class.new(nodifier, *args)
+      end
+
+      def inspect
+        "<##{self.class} upstreams=[#{upstreams.map(&:inspect).join(', ')}] arguments=[#{arguments.map(&:inspect).join(', ')}]>"
       end
 
       def evaluate(db)
@@ -53,106 +253,251 @@ module ConceptQL
         evaluate(db).sql
       end
 
-      def select_it(query, specific_type = nil)
-        specific_type = type if specific_type.nil? && respond_to?(:type)
-        q = query.select(*columns(query, specific_type))
+      def optimized
+        dup_values(values.map{|x| x.is_a?(Operator) ? x.optimized : x})
+      end
+
+      def unionable?(other)
+        false
+      end
+
+      def select_it(query, specific_domain = nil)
+        if specific_domain.nil? && respond_to?(:domain) && TABLE_COLUMNS.keys.include?(domain)
+          specific_domain = domain
+        end
+
+        q = setup_select(query, specific_domain)
+
         if scope && scope.person_ids && upstreams.empty?
           q = q.where(person_id: scope.person_ids).from_self
         end
+
         q
       end
 
-      def types
-        @types ||= determine_types
+      def domains(db)
+        @domains ||= determine_domains(db)
       end
 
       def stream
         @stream ||= upstreams.first
       end
 
-      def columns(query, local_type = nil)
-        criterion_type = :criterion_type
-        if local_type
-          criterion_type = Sequel.cast_string(local_type.to_s).as(:criterion_type)
+      def setup_select(query, local_domain = nil)
+        query = modify_query(query, local_domain)
+        query.select(*columns(query, local_domain))
+      end
+
+      def columns(query, local_domain)
+        criterion_domain = :criterion_domain
+
+        if local_domain
+          criterion_domain = cast_column(:criterion_domain, local_domain.to_s)
         end
+
         columns = [:person_id,
-                    type_id(local_type),
-                    criterion_type]
-        columns += date_columns(query, local_type)
-        columns += value_columns(query, local_type)
+                    domain_id(local_domain),
+                    criterion_domain]
+        columns += date_columns(query, local_domain)
+        columns += [ source_value(query, local_domain) ]
+        columns += additional_columns(query, local_domain)
       end
 
       def label
-        options[:label]
+        @label ||= begin
+          options.delete(:label) if options[:label] && options[:label].to_s.strip.empty?
+          options[:label].respond_to?(:strip) ? options[:label].strip : options[:label]
+        end
+      end
+
+      attr :errors, :warnings
+
+      def valid?(db, opts = {})
+        return @errors.empty? if defined?(@errors)
+        @errors = []
+        @warnings = []
+        validate(db, opts)
+        errors.empty?
+      end
+
+      def upstreams_valid?(db, opts = {})
+          valid?(db, opts) && upstreams.all?{|u| u.upstreams_valid?(db, opts)}
+      end
+
+      def scope
+        nodifier.scope
+      end
+
+      def data_model
+        nodifier.data_model
+      end
+
+      def database_type
+        nodifier.database_type
+      end
+
+      def cast_column(column, value = nil)
+        type = Scope::COLUMN_TYPES.fetch(column)
+        case type
+        when String, :String
+          Sequel.cast_string(value).as(column)
+        when Date, :Date
+          Sequel.cast(value, type).as(column)
+        when Float, :Bigint, :Float
+          Sequel.cast_numeric(value, type).as(column)
+        else
+          raise "Unexpected type: '#{type.inspect}' for column: '#{column}'"
+        end
       end
 
       private
-      attr :scope
+
+      def annotate_values(db, opts)
+        (upstreams.map { |op| op.annotate(db, opts) } + arguments).push(options)
+      end
 
       def criterion_id
         :criterion_id
       end
 
-      def type_id(type = nil)
-        return :criterion_id if type.nil?
-        type = :person if type == :death
-        Sequel.expr(make_type_id(type)).as(:criterion_id)
+      def domain_id(domain = nil)
+        return :criterion_id if domain.nil?
+        domain = :person if domain == :death
+        Sequel.expr(make_domain_id(domain)).as(:criterion_id)
       end
 
-      def make_type_id(type)
-        (type.to_s + '_id').to_sym
+      def make_domain_id(domain)
+        (domain.to_s + '_id').to_sym
       end
 
       def make_table_name(table)
         "#{table}___tab".to_sym
       end
 
-      def value_columns(query, type)
-        [
-          numeric_value(query),
-          string_value(query),
-          concept_id_value(query),
-          units_source_value(query),
-          source_value(query, type)
-        ]
+      def query_cols
+        raise NotImplementedError, self
+      end
+
+      def query_columns(query)
+        unless cols = query.opts[:force_columns]
+          cols = query_cols
+        end
+
+        if ENV['CONCEPTQL_CHECK_COLUMNS']
+          if cols.sort != query.columns.sort
+            raise "columns don't match:\nclass: #{self.class}\nexpected: #{cols}\nactual: #{query.columns}\nvalues: #{values}\nSQL: #{query.sql}"
+          end
+        end
+
+        cols
+      end
+
+      def omopv4_plus?
+        data_model == :omopv4_plus
       end
 
-      def numeric_value(query)
-        return :value_as_number if query.columns.include?(:value_as_number)
-        Sequel.cast_numeric(nil, Float).as(:value_as_number)
+      def omopv4?
+        data_model == :omopv4
       end
 
-      def string_value(query)
-        return :value_as_string if query.columns.include?(:value_as_string)
-        Sequel.cast_string(nil).as(:value_as_string)
+      def impala?
+        database_type.to_sym == :impala
       end
 
-      def concept_id_value(query)
-        return :value_as_concept_id if query.columns.include?(:value_as_concept_id)
-        Sequel.cast_numeric(nil).as(:value_as_concept_id)
+      def table_to_sym(table)
+        case table
+        when Symbol
+          table = Sequel.split_symbol(table)[1].to_sym
+        end
+        table
       end
 
-      def units_source_value(query)
-        return :units_source_value if query.columns.include?(:units_source_value)
-        Sequel.cast_string(nil).as(:units_source_value)
+      def table_cols(table)
+        table = table_to_sym(table)
+        cols = TABLE_COLUMNS.fetch(table)
+        if omopv4_plus?
+          cols += Array(table_vocabulary_id(table))
+        end
+        cols
       end
 
-      def source_value(query, type)
-        return :source_value if query.columns.include?(:source_value)
-        Sequel.cast_string(source_value_column(query, type)).as(:source_value)
+      def table_columns(*tables)
+        tables.map{|t| table_cols(t)}.flatten
       end
 
-      def date_columns(query, type = nil)
-        return [:start_date, :end_date] if (query.columns.include?(:start_date) && query.columns.include?(:end_date))
-        return [:start_date, :end_date] unless type
-        sd = start_date_column(query, type)
-        sd = Sequel.expr(sd).cast(:date).as(:start_date) unless sd == :start_date
-        ed = end_date_column(query, type)
-        ed = Sequel.function(:coalesce, Sequel.expr(ed).cast(:date), Sequel.expr(start_date_column(query, type))).as(:end_date) unless ed == :end_date
+      def table_source_value(table)
+        TABLE_SOURCE_VALUE_COLUMN.fetch(table_to_sym(table))
+      end
+
+      def table_vocabulary_id(table)
+        TABLE_VOCABULARY_ID_COLUMN[table_to_sym(table)]
+      end
+
+      def additional_columns(query, domain)
+        special_columns = {
+          provenance_type: Proc.new { provenance_type(query, domain) },
+          provider_id: Proc.new { provider_id(query, domain) },
+          place_of_service_concept_id: Proc.new { place_of_service_concept_id(query, domain) }
+        }
+
+        additional_cols = special_columns.each_with_object([]) do |(column, proc_obj), columns|
+          columns << proc_obj.call if dynamic_columns.include?(column)
+        end
+
+        standard_columns = dynamic_columns - Scope::DEFAULT_COLUMNS.keys
+        standard_columns -= special_columns.keys
+
+        standard_columns.each do |column|
+          additional_cols << if query_columns(query).include?(column)
+            column
+          else
+            cast_column(column)
+          end
+        end
+
+        additional_cols
+      end
+
+      def source_value(query, domain)
+        return :source_value if query_columns(query).include?(:source_value)
+        cast_column(:source_value, source_value_column(query, domain))
+      end
+
+      def provenance_type(query, domain)
+        return :provenance_type if query_columns(query).include?(:provenance_type)
+        cast_column(:provenance_type, provenance_type_column(query, domain))
+      end
+
+      def provider_id(query, domain)
+        return :provider_id if query_columns(query).include?(:provider_id)
+        cast_column(:provider_id, provider_id_column(query, domain))
+      end
+
+      def place_of_service_concept_id(query, domain)
+        return :place_of_service_concept_id if query_columns(query).include?(:place_of_service_concept_id)
+        cast_column(:place_of_service_concept_id, place_of_service_concept_id_column(query, domain))
+      end
+
+      def date_columns(query, domain = nil)
+        return [:start_date, :end_date] if (query_columns(query).include?(:start_date) && query_columns(query).include?(:end_date))
+        return [:start_date, :end_date] unless domain
+
+        date_klass = Date
+        if query.db.database_type == :impala
+          date_klass = DateTime
+        end
+
+        sd = start_date_column(query, domain)
+        sd = Sequel.cast(Sequel.expr(sd), date_klass).as(:start_date) unless sd == :start_date
+        ed = end_date_column(query, domain)
+        ed = Sequel.cast(Sequel.function(:coalesce, Sequel.expr(ed), start_date_column(query, domain)), date_klass).as(:end_date) unless ed == :end_date
         [sd, ed]
       end
 
-      def start_date_column(query, type)
+      # TODO: Move these hashes into a configuration file
+      # THey are OMOP-specific bits of information and need to be abstracted away
+
+      def start_date_column(query, domain)
         {
           condition_occurrence: :condition_start_date,
           death: :death_date,
@@ -165,10 +510,10 @@ module ConceptQL
           observation: :observation_date,
           observation_period: :observation_period_start_date,
           visit_occurrence: :visit_start_date
-        }[type]
+        }[domain]
       end
 
-      def end_date_column(query, type)
+      def end_date_column(query, domain)
         {
           condition_occurrence: :condition_end_date,
           death: :death_date,
@@ -181,10 +526,10 @@ module ConceptQL
           observation: :observation_date,
           observation_period: :observation_period_end_date,
           visit_occurrence: :visit_end_date
-        }[type]
+        }[domain]
       end
 
-      def source_value_column(query, type)
+      def source_value_column(query, domain)
         {
           condition_occurrence: :condition_source_value,
           death: :cause_of_death_source_value,
@@ -197,7 +542,61 @@ module ConceptQL
           observation: :observation_source_value,
           observation_period: nil,
           visit_occurrence: :place_of_service_source_value
-        }[type]
+        }[domain]
+      end
+
+      def provenance_type_column(query, domain)
+        {
+          condition_occurrence: :condition_type_concept_id,
+          death: :death_type_concept_id,
+          drug_exposure: :drug_type_concept_id,
+          observation: :observation_type_concept_id,
+          procedure_occurrence: :procedure_type_concept_id
+        }[domain]
+      end
+
+      def provenance_type_column(query, domain)
+        {
+          condition_occurrence: :condition_type_concept_id,
+          death: :death_type_concept_id,
+          drug_exposure: :drug_type_concept_id,
+          observation: :observation_type_concept_id,
+          procedure_occurrence: :procedure_type_concept_id
+        }[domain]
+      end
+
+      def provider_id_column(query, domain)
+        {
+          condition_occurrence: :associated_provider_id,
+          death: :death_type_concept_id,
+          drug_exposure: :prescribing_provider_id,
+          observation: :associated_provider_id,
+          person: :provider_id,
+          procedure_occurrence: :associated_provider_id,
+          provider: :provider_id
+        }[domain]
+      end
+
+      def place_of_service_concept_id_column(query, domain)
+        return nil if domain.nil?
+        return :place_of_service_concept_id if table_cols(domain).include?(:visit_occurrence_id)
+        return nil
+      end
+
+      def modify_query(query, domain)
+        {
+          place_of_service_concept_id: ConceptQL::QueryModifiers::PoSQueryModifier,
+          drug_name: ConceptQL::QueryModifiers::DrugQueryModifier
+        }.each do |column, klass|
+          #p [domain, column, table, join_id, source_column]
+          #p dynamic_columns
+          #p query_cols
+          next if domain.nil?
+          next unless dynamic_columns.include?(column)
+          query = klass.new(query, self).modified_query
+        end
+
+        query
       end
 
       def person_date_of_birth(query)
@@ -206,31 +605,165 @@ module ConceptQL
 
       def assemble_date(query, *symbols)
         strings = symbols.map do |symbol|
-          Sequel.cast_string(Sequel.function(:coalesce, Sequel.cast_string(symbol), Sequel.cast_string('01')))
+          sub = '2000'
+          col = Sequel.cast_string(symbol)
+          if symbol != :year_of_birth
+            sub = '01'
+            col = Sequel.function(:lpad, col, 2, '0')
+          end
+          Sequel.function(:coalesce, col, Sequel.expr(sub))
         end
-        strings = strings.zip(['-'] * (symbols.length - 1)).flatten.compact
-        concatted_strings = Sequel.join(strings)
-        case query.db.database_type
+
+        strings_with_dashes = strings.zip(['-'] * (symbols.length - 1)).flatten.compact
+        concatted_strings = Sequel.join(strings_with_dashes)
+
+        date = concatted_strings
+        if query.db.database_type == :impala
+          date = Sequel.cast(Sequel.function(:concat_ws, '-', *strings), DateTime)
+        end
+        cast_date(query.db, date)
+      end
+
+      def cast_date(db, date)
+        case db.database_type
         when :oracle
-          Sequel.function(:to_date, concatted_strings, 'YYYY-MM-DD')
+          Sequel.function(:to_date, date, 'YYYY-MM-DD')
         when :mssql
-          Sequel.lit('CONVERT(DATETIME, ?)', concatted_strings)
+          Sequel.lit('CONVERT(DATETIME, ?)', date)
         else
-          Sequel.cast(concatted_strings, Date)
+          Sequel.cast(date, Date)
         end
       end
 
-      def determine_types
+      def determine_domains(db)
         if upstreams.empty?
-          if respond_to?(:type)
-            [type]
+          if respond_to?(:domain)
+            [domain]
           else
-            raise "Operator doesn't seem to specify any type"
+            [:invalid]
           end
         else
-          upstreams.map(&:types).flatten.uniq
+          doms = upstreams.compact.flat_map { |up| up.domains(db) }.uniq
+          doms.empty? ? [:invalid] : doms
         end
       end
+
+      # Validation Related
+
+      def upstream_operator_names
+        @upstreams.map(&:operator_name)
+      end
+
+      def validate(db, opts = {})
+        add_error("invalid label") if label && !label.is_a?(String)
+        self.class.validations.each do |args|
+          send(*args)
+        end
+      end
+
+      def validate_no_upstreams
+        add_error("has upstreams", upstream_operator_names) unless @upstreams.empty?
+      end
+
+      def validate_one_upstream
+        validate_at_least_one_upstream
+        validate_at_most_one_upstream
+      end
+
+      def validate_at_most_one_upstream
+        add_error("has multiple upstreams", upstream_operator_names) if @upstreams.length > 1
+      end
+
+      def validate_at_least_one_upstream
+        add_error("has no upstream") if @upstreams.empty?
+      end
+
+      def validate_no_arguments
+        add_error("has arguments", @arguments) unless @arguments.empty?
+      end
+
+      def validate_one_argument
+        validate_at_least_one_argument
+        validate_at_most_one_argument
+      end
+
+      def validate_at_most_one_argument
+        add_error("has multiple arguments", @arguments) if @arguments.length > 1
+      end
+
+      def validate_at_least_one_argument
+        add_error("has no arguments") if @arguments.empty?
+      end
+
+      def validate_option(format, *opts)
+        opts.each do |opt|
+          if options.has_key?(opt)
+            unless format === options[opt]
+              add_error("wrong option format", opt.to_s)
+            end
+          end
+        end
+      end
+
+      def validate_required_options(*opts)
+        opts.each do |opt|
+          unless options.has_key?(opt)
+            add_error("required option not present", opt.to_s)
+          end
+        end
+      end
+
+      def bad_arguments
+        return [] unless self.class.codes_regexp
+        @bad_arguments ||= arguments.reject do |arg|
+          self.class.codes_regexp === arg
+        end
+      end
+
+      def validate_codes_match
+        unless bad_arguments.empty?
+          add_warning("improperly formatted code", *bad_arguments)
+        end
+      end
+
+      def add_warnings?(db, opts = {})
+        @errors.empty? && db && db.adapter_scheme != :mock && !opts[:skip_db]
+      end
+
+      def add_error(*args)
+        errors << args
+      end
+
+      def add_warning(*args)
+        warnings << args
+      end
+
+      def needs_arguments_cte?(args)
+        impala? && arguments.length > 5000
+      end
+
+      def arguments_fix(db, args = nil)
+        args ||= arguments
+        return args unless needs_arguments_cte?(args)
+        args = args.dup
+        first_arg = Sequel.expr(args.shift).as(:arg)
+        args.unshift(first_arg)
+        args = args.map { |v| [v] }
+        args_cte = db.values(args)
+        db[:args]
+          .with(:args, args_cte)
+          .select(:arg)
+      end
+
+      def include_counts?(db, opts)
+        db && !opts[:skip_db] && !opts[:skip_counts]
+      end
     end
   end
 end
+
+# Require all operator subclasses eagerly
+Dir.new(File.dirname(__FILE__)).
+  entries.
+  each{|filename| require_relative filename if filename =~ /\.rb\z/ && filename != File.basename(__FILE__)}
+ConceptQL::Operators.operators.values.each(&:freeze)