conceptql 0.2.0 → 0.3.0

data/lib/conceptql/operators/recall.rb

@@ -10,28 +10,63 @@ module ConceptQL
     # This operator will look for a sub-concept that has been created through the
     # "define" operator and will fetch the results cached in the corresponding table
     class Recall < Operator
+      register __FILE__
+
       desc <<-EOF
 Recalls a set of named results that were previously stored using the Define operator.
 Must be surrounded by the same Let operator as surrounds the corresponding Define operator.
       EOF
       argument :name, type: :string
-      category 'Variable Assignment'
+      category "Get Related Data"
+      basic_type :selection
+      validate_no_upstreams
+      validate_one_argument
+      default_query_columns
 
       def query(db)
         scope.from(db, source)
       end
 
-      def columns(query, local_type)
-        COLUMNS
-      end
-
-      def types
-        scope.types(source)
+      def domains(db)
+        scope.domains(source, db)
       end
 
       def source
         arguments.first
       end
+
+      def annotate(db, opts = {})
+        @annotate ||= if valid?(db) && replaced?
+          original.annotate(db, opts)
+        else
+          super
+        end
+      end
+
+      def original
+        nodifier.scope.fetch_operator(source)
+      end
+
+      private
+
+      def validate(db, opts = {})
+        super
+        if arguments.length == 1
+          if scope.fetch_operator(source)
+            scope.recall_dependencies[source].each do |d|
+              if scope.recall_dependencies[d].include?(source)
+                add_error("mutually referential recalls", d)
+              end
+            end
+          else
+            add_error("no matching label", source)
+          end
+        end
+      end
+
+      def replaced?
+        options[:replaced]
+      end
     end
   end
 end

data/lib/conceptql/operators/revenue_code.rb

@@ -0,0 +1,40 @@
+require_relative 'standard_vocabulary_operator'
+
+module ConceptQL
+  module Operators
+    class RevenueCode < StandardVocabularyOperator
+      register __FILE__
+
+      desc 'Searches the procedure_occurrence table for all procedures that have an associated procedure_cost record with matching revenue codes'
+      argument :revenue_codes, type: :codelist, vocab: 'Revenue Code'
+      predominant_domains :procedure_occurrence
+
+      def query(db)
+        costs = super(db).select(:procedure_occurrence_id)
+        db[:procedure_occurrence].where(procedure_occurrence_id: costs)
+      end
+
+      def domain
+        :procedure_occurrence
+      end
+
+      def table
+        :procedure_cost
+      end
+
+      def vocabulary_id
+        43
+      end
+
+      def concept_column
+        :revenue_code_concept_id
+      end
+
+      def code_column
+        :revenue_code_source_value
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/operators/rxnorm.rb

@@ -1,11 +1,15 @@
 require_relative 'standard_vocabulary_operator'
+require_relative '../behaviors/drugish'
 
 module ConceptQL
   module Operators
     class Rxnorm < StandardVocabularyOperator
+      register __FILE__
+
       preferred_name 'RxNorm'
-      desc 'Finds all drug_exposures by RxNorm codes'
       argument :rxnorms, type: :codelist, vocab: 'RxNorm'
+      predominant_domains :drug_exposure
+      include ConceptQL::Drugish
 
       def table
         :drug_exposure

data/lib/conceptql/operators/snomed.rb

@@ -3,10 +3,11 @@ require_relative 'standard_vocabulary_operator'
 module ConceptQL
   module Operators
     class Snomed < StandardVocabularyOperator
+      register __FILE__
+
       preferred_name 'SNOMED'
-      desc 'Find all condition_occurrences by SNOMED codes'
       argument :snomeds, type: :codelist, vocab: 'SNOMED'
-      predominant_types :condition_occurrence
+      predominant_domains :condition_occurrence
 
       def table
         :condition_occurrence

data/lib/conceptql/operators/source_vocabulary_operator.rb

@@ -1,4 +1,4 @@
-require_relative 'operator'
+require_relative 'vocabulary_operator'
 
 module ConceptQL
   module Operators
@@ -25,28 +25,58 @@ module ConceptQL
     # * vocabulary_id
     #   * The vocabulary ID of the source vocabulary for the criterion
     #   * e.g. for ICD-9, a value of 2 (for ICD-9-CM)
-    class SourceVocabularyOperator < Operator
-      category 'Source Vocabulary'
-      category 'Code Lists'
+    class SourceVocabularyOperator < VocabularyOperator
 
       def query(db)
-        db.from(table_name)
-          .join(:vocabulary__source_to_concept_map___scm, scm__target_concept_id: table_concept_column)
-          .where(Sequel.expr(scm__source_code: values, scm__source_vocabulary_id: vocabulary_id).&(Sequel.expr(scm__source_code: table_source_column)))
+        ds = db.from(table_name).where(conditions(db))
+        if omopv4?
+          ds = ds.join(:source_to_concept_map___scm, [[:scm__target_concept_id, table_concept_column], [:scm__source_code, table_source_column]])
+        end
+        ds
       end
 
-      def type
-        table
+      def query_cols
+        if omopv4?
+          table_columns(table_name, :source_to_concept_map)
+        else
+          table_columns(table_name)
+        end
       end
 
-      private
+      def unionable?(other)
+        other.is_a?(self.class)
+      end
+
+      def union(other)
+        dup_values(values + other.values)
+      end
 
-      def table_name
-        @table_name ||= make_table_name(table)
+      def conditions(db)
+        if omopv4?
+          [[:scm__source_code, arguments_fix(db)], [:scm__source_vocabulary_id, vocabulary_id]]
+        else
+          conditions = { code_column => arguments_fix(db) }
+          conditions[vocabulary_id_column] = vocabulary_id if vocabulary_id_column
+          conditions
+        end
       end
 
-      def table_concept_column
-        "tab__#{concept_column}".to_sym
+      def describe_code(db, code)
+        db[:source_to_concept_map].filter(:source_vocabulary_id => vocabulary_id).filter(:source_code => code).map(:source_code_description)[0]
+      end
+
+      private
+
+      def validate(db, opts = {})
+        super
+        if add_warnings?(db, opts)
+          args = arguments.dup
+          args -= bad_arguments
+          missing_args = args - db[:source_to_concept_map].where(:source_vocabulary_id=>vocabulary_id, :source_code=>arguments_fix(db, args)).select_map(:source_code)
+          unless missing_args.empty?
+            add_warning("unknown source code", *missing_args)
+          end
+        end
       end
 
       def table_source_column

data/lib/conceptql/operators/standard_vocabulary_operator.rb

@@ -1,4 +1,4 @@
-require_relative 'operator'
+require_relative 'vocabulary_operator'
 
 module ConceptQL
   module Operators
@@ -6,7 +6,7 @@ module ConceptQL
     #
     # If that seems confusing, then think of CPT or SNOMED criteria.  That type of criterion takes a set of values that live in the OMOP concept table.
     #
-    # My coworker came up with a nice, gneralized query that checks for matching concept_ids and matching source_code values.  This class encapsulates that query.
+    # My coworker came up with a nice, generalized query that checks for matching concept_ids and matching source_code values.  This class encapsulates that query.
     #
     # Subclasses must provide the following methods:
     # * table
@@ -18,26 +18,47 @@ module ConceptQL
     # * vocabulary_id
     #   * The vocabulary ID of the source vocabulary for the criterion
     #   * e.g. for CPT, a value of 4 (for CPT-4)
-    class StandardVocabularyOperator < Operator
-      category 'Standard Vocabulary'
-      category 'Code Lists'
+    class StandardVocabularyOperator < VocabularyOperator
+
       def query(db)
-        db.from(table_name)
-          .join(:vocabulary__concept___c, c__concept_id: table_concept_column)
-          .where(c__concept_code: values, c__vocabulary_id: vocabulary_id)
+        ds = db.from(table_name)
+          .where(conditions(db))
+        if omopv4?
+          ds = ds.join(:concept___c, c__concept_id: table_concept_column)
+        end
+        ds
       end
 
-      def type
-        table
+      def query_cols
+        table_columns(table_name, :concept)
+      end
+
+      def conditions(db)
+        if omopv4?
+          {c__concept_code: arguments_fix(db), c__vocabulary_id: vocabulary_id}
+        else
+          conditions = { code_column => arguments_fix(db) }
+          conditions[vocabulary_id_column] = vocabulary_id if vocabulary_id_column
+          conditions
+        end
       end
-      private
 
-      def table_name
-        @table_name ||= make_table_name(table)
+      def describe_code(db, code)
+        db[:concept].filter(:vocabulary_id => vocabulary_id).filter(:concept_code => code).map(:concept_name)[0]
       end
 
-      def table_concept_column
-        "tab__#{concept_column}".to_sym
+      private
+
+      def validate(db, opts = {})
+        super
+        if add_warnings?(db, opts)
+          args = arguments.dup
+          args -= bad_arguments
+          missing_args = args - db[:concept].where(:vocabulary_id=>vocabulary_id, :concept_code=>arguments_fix(db, args)).select_map(:concept_code)
+          unless missing_args.empty?
+            add_warning("unknown concept code", *missing_args)
+          end
+        end
       end
     end
   end

data/lib/conceptql/operators/started_by.rb

@@ -3,8 +3,9 @@ require_relative 'temporal_operator'
 module ConceptQL
   module Operators
     class StartedBy < TemporalOperator
+      register __FILE__
       desc <<-EOF
-If LHR has the same start date as RHR, but LHR's end_date falls on or after end_date of RHR, LHR is passed through.
+If a result in the left hand results (LHR) has the same start_date and the same or a later end_date as a result in the right hand results (RHR), it is passed through.
 L----Y----L
 R-------R
 L--N--L

data/lib/conceptql/operators/sum.rb

@@ -3,14 +3,20 @@ require_relative 'pass_thru'
 module ConceptQL
   module Operators
     class Sum < PassThru
+      register __FILE__
+
       desc <<-EOF
 Sums value_as_number across all results that match on all but start_date, end_date.
 For start_date and end_date the min and max of each respectively is returned.'
       EOF
+      default_query_columns
+      validate_at_least_one_upstream
+      validate_no_arguments
+      require_column :value_as_number
 
       def query(db)
         db.from(unioned(db))
-          .select_group(*(COLUMNS - [:start_date, :end_date, :criterion_id, :value_as_number]))
+          .select_group(*(dynamic_columns - [:start_date, :end_date, :criterion_id, :value_as_number]))
           .select_append(Sequel.lit('?', 0).as(:criterion_id))
           .select_append{ min(start_date).as(:start_date) }
           .select_append{ max(end_date).as(:end_date) }

data/lib/conceptql/operators/temporal_operator.rb

@@ -1,21 +1,87 @@
 require_relative 'binary_operator_operator'
+require_relative '../date_adjuster'
 
 module ConceptQL
   module Operators
     # Base class for all temporal operators
     #
     # Subclasses must implement the where_clause method which should probably return
-    # a proc that can be executed as a Sequel "virtual row" e.g.
-    # Proc.new { l.end_date < r.start_date }
+    # a Sequel expression to use for filtering.
     class TemporalOperator < BinaryOperatorOperator
       reset_categories
-      category %w(Temporal Relative)
+      category "Filter by Comparing"
+      default_query_columns
+
+      option :within, type: :string, instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+      option :at_least, type: :string, instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+      option :occurrences, type: :integer, desc: "Number of occurrences that must precede the event of interest, e.g. if you'd like the 4th event in a set of events, set occurrences to 3"
+
+      validate_option DateAdjuster::VALID_INPUT, :within, :at_least
+      validate_option /\A\d+\z/, :occurrences
+
+      def self.within_skip(type)
+        define_method(:"within_check_#{type}?"){false}
+      end
 
       def query(db)
-        db.from(db.from(left_stream(db))
-                  .join(right_stream(db), l__person_id: :r__person_id)
-                  .where(where_clause)
-                  .select_all(:l))
+        ds = db.from(left_stream(db))
+               .join(right_stream(db), l__person_id: :r__person_id)
+               .where(where_clause)
+               .select_all(:l)
+
+        ds = add_option_conditions(ds)
+        ds.from_self
+      end
+
+      def add_option_conditions(ds)
+        if within = options[:within]
+          ds = add_within_condition(ds, within)
+        end
+
+        if at_least = options[:at_least]
+          ds = add_within_condition(ds, at_least, :exclude)
+        end
+
+        if occurrences = options[:occurrences]
+          ds = add_occurrences_condition(ds, occurrences)
+        end
+
+        ds
+      end
+
+      def add_within_condition(ds, within, meth=:where)
+        within = DateAdjuster.new(within)
+        after = within.adjust(:r__start_date, true)
+        before = within.adjust(:r__end_date)
+        within_col = Sequel.expr(within_column)
+        ds = ds.send(meth){within_col >= after} if within_check_after?
+        ds = ds.send(meth){within_col <= before} if within_check_before?
+        ds.distinct
+      end
+
+      def add_occurrences_condition(ds, occurrences)
+        occurrences_col = occurrences_column
+        ds.distinct.from_self
+          .select_append{row_number{}.over(:partition => :person_id, :order => occurrences_col).as(:occurrence)}
+          .from_self
+          .select(*query_columns(ds))
+          .where{occurrence > occurrences.to_i}
+      end
+
+      def within_column
+        :l__start_date
+      end
+
+      def occurrences_column
+        :start_date
+      end
+
+      def within_check_after?
+        true
+      end
+
+      def within_check_before?
+        true
       end
 
       def inclusive?
@@ -23,11 +89,11 @@ module ConceptQL
       end
 
       def left_stream(db)
-        Sequel.expr(left.evaluate(db)).as(:l)
+        Sequel.expr(left.evaluate(db).from_self).as(:l)
       end
 
       def right_stream(db)
-        Sequel.expr(right.evaluate(db)).as(:r)
+        Sequel.expr(right.evaluate(db).from_self).as(:r)
       end
     end
   end

data/lib/conceptql/operators/time_window.rb

@@ -20,19 +20,26 @@ module ConceptQL
     # pass '', '0', or nil as that argument.  E.g.:
     # start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
     class TimeWindow < Operator
-      desc 'Adjusts the start_date and end_date columns to create a new window of time for each result.'
+      register __FILE__
+
+      desc 'Adjusts the start_date and end_date to create a new window of time for each result.'
       option :start, type: :string
       option :end, type: :string
       allows_one_upstream
-      category %(Temporal Manipulation)
+      validate_one_upstream
+      validate_no_arguments
+      validate_option DateAdjuster::VALID_INPUT, :start, :end
+      category "Modify Data"
+      basic_type :temporal
+      default_query_columns
 
       def query(db)
-        db.extension :date_arithmetic
         db.from(stream.evaluate(db))
       end
 
       private
-      def date_columns(query, type = nil)
+
+      def date_columns(query, domain = nil)
         [adjusted_start_date, adjusted_end_date]
       end
 
@@ -44,26 +51,9 @@ module ConceptQL
         adjusted_date(:end, :end_date)
       end
 
-      def manipulator
-        @manipulator ||= options.fetch(:date_manipulator, Sequel)
-      end
-
-      # NOTE: This produces PostgreSQL-specific date adjustment.  I'm not yet certain how to generalize this
-      # or make different versions based on RDBMS
       def adjusted_date(option_arg, column)
-        arg = options[option_arg]
-        arg ||= ''
-        return ['end_date', column].join('___').to_sym if arg.downcase == 'end'
-        return ['start_date', column].join('___').to_sym if arg.downcase == 'start'
-        return Sequel.cast(Date.parse(arg).strftime('%Y-%m-%d'), Date).as(column) if arg =~ /^\d{4}-\d{2}-\d{2}$/
-        adjusted_date = DateAdjuster.new(arg).adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
-          if quantity > 0
-            manipulator.date_add(sql, units => quantity)
-          else
-            manipulator.date_sub(sql, units => quantity.abs)
-          end
-        end
-        Sequel.cast(adjusted_date, Date).as(column)
+        adjusted_date = DateAdjuster.new(options[option_arg], manipulator: options[:manipulator]).adjust(column)
+        adjusted_date.as(column)
       end
     end
   end