conceptql 0.2.0 → 0.4.0

data/lib/conceptql/operators/pass_thru.rb

@@ -1,10 +1,24 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
+    # The base class of operators whose domains and columns are those of their
+    # upstreams (union, count, sum, numeric, dedup, episode, time_window and
+    # co_reported). It has no query method, so it does not register: pass_thru
+    # is not an operator name.
     class PassThru < Operator
-      def types
-        upstreams.map(&:types).flatten.uniq
+      basic_type :set
+      no_desc
+
+      def domains(db)
+        doms = upstreams.compact.flat_map { |up| up.domains(db) }.uniq
+        doms.empty? ? [ :invalid ] : doms
+      end
+
+      def query_cols
+        upstreams.map(&:query_cols).inject(:|)
       end
     end
   end

data/lib/conceptql/operators/person.rb

@@ -1,27 +1,39 @@
-require_relative 'casting_operator'
+# frozen_string_literal: true
+
+require_relative "casting_operator"
 
 module ConceptQL
   module Operators
     class Person < CastingOperator
-      desc 'Returns all people in the database, or if given a upstream, converts all results to the set of patients contained in those results.'
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      register __FILE__
+
+      desc "Selects all records in the person table."
+
       allows_one_upstream
-      types :person
+      domains :person
 
-      def my_type
+      def my_domain
         :person
       end
 
+      def source_table
+        dm.table_by_domain(my_domain)
+      end
+
       def i_point_at
         []
       end
 
       def these_point_at_me
-        # I could list ALL the types we use, but the default behavior of casting,
+        # I could list ALL the domains we use, but the default behavior of casting,
         # when there is no explicit casting defined, is to convert everything to
         # person IDs
         #
         # So by defining no known castable relationships in this operator, all
-        # types will be converted to person
+        # domains will be converted to person
         []
       end
     end

data/lib/conceptql/operators/person_filter.rb

@@ -1,12 +1,18 @@
-require_relative 'binary_operator_operator'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
 
 module ConceptQL
   module Operators
     class PersonFilter < BinaryOperatorOperator
-      desc 'Only passes through a result from the LHR if the person appears in the RHR.'
+      register __FILE__
+
+      desc "Passes along left hand records with a corresponding person_id in the right hand records."
+      default_query_columns
+
       def query(db)
         db.from(left.evaluate(db))
-          .where(person_id: right.evaluate(db).select_group(:person_id))
+          .where(person_id: right.evaluate(db).from_self.select_group(:person_id))
       end
     end
   end

data/lib/conceptql/operators/place_of_service_filter.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    # Filters the incoming stream of records to only those that have a
+    # an associated visit_occurrence with the matching place_of_service
+    #
+    # Provenance related concepts are the ones found in the xxx_type_concept_id
+    # field.
+    #
+    # If the record has NULL for the provenance-related field, they are filtered
+    # out.
+    #
+    # Multiple provenances can be specified at once
+    class PlaceOfServiceFilter < Operator
+      register __FILE__
+
+      desc "Passes along records that match one or more of the Medicare Place Of Service values."
+
+      argument :places_of_service, type: :codelist, vocab: "Place of Service"
+      category "Filter Single Stream"
+      basic_type :temporal
+      allows_one_upstream
+      validate_one_upstream
+      validate_at_least_one_argument
+      require_column :visit_source_concept_id
+      default_query_columns
+
+      def query(db)
+        db.from(stream.evaluate(db))
+          .where(visit_source_concept_id: place_of_service_concept_ids(db))
+      end
+
+      private
+
+      def place_of_service_concept_ids(db)
+        dm.concept_ids(db, "Place of Service", arguments.map(&:to_s))
+      end
+    end
+  end
+end

data/lib/conceptql/operators/provenance.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+require_relative "../behaviors/provenanceable"
+
+module ConceptQL
+  module Operators
+    # Filters the incoming stream of records to only those that have a
+    # provenance-related concept_id.
+    #
+    # Provenance related concepts are the ones found in the xxx_type_concept_id
+    # field.
+    #
+    # If the record has NULL for the provenance-related field, they are filtered
+    # out.
+    #
+    # Multiple provenances can be specified at once
+    class Provenance < Operator
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
+
+      desc "Passes along records with the indicated provenance (e.g. inpatient, outpatient, file type)."
+
+      argument :provenance_types, label: "Provenance Types", type: :codelist
+      category "Filter Single Stream"
+      basic_type :temporal
+      allows_one_upstream
+      validate_one_upstream
+      default_query_columns
+
+      include ConceptQL::Provenanceable
+
+      attr_reader :db
+
+      def query(db)
+        @db = db
+        db.from(stream.evaluate(db)).where(build_where_from_codes(db, arguments))
+      end
+
+      private
+
+      def additional_validation(db, _opts = {})
+        # With no lexicon there are no provenance concepts to check against,
+        # and every keyword would be reported as unrecognized. Vocabulary
+        # operators skip their unknown-code check in the same situation.
+        return if dm.lexicon.strategy == :no_db
+
+        bad_keywords = find_bad_keywords(db, arguments)
+        return unless bad_keywords.present?
+
+        add_error("unrecognized keywords", *bad_keywords.uniq)
+      end
+    end
+  end
+end

data/lib/conceptql/operators/provider_filter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    # Filters incoming records to only those that have been associated with
+    # providers matching the given criteria.
+    class ProviderFilter < Operator
+      register __FILE__
+
+      desc "Passes along records where the provider's specialty matches a given set of specialty_concept_ids.\n\nA single record may have multiple specialties, so the operator will return all records that match any of the given specialties, which may lead to duplicate records."
+      option :specialties, type: :string
+      option :roles, type: :string
+      category "Filter Single Stream"
+      basic_type :temporal
+      allows_one_upstream
+      validate_one_upstream
+      validate_required_options :specialties, :roles
+      require_column :provider_id
+      require_column :context_specialty_concept_id
+      require_column :practitioner_specialty_concept_id
+      require_column :role_type_concept_id
+      default_query_columns
+
+      def query(db)
+        ds = stream.evaluate(db).from_self(alias: :upstream)
+
+        specialty_concept_ids = options[:specialties].split(/\s*,\s*/)
+        role_type_concept_ids = options[:roles].split(/\s*,\s*/)
+
+        unless specialty_concept_ids.include?("*")
+          spec_con_ids = specialty_concept_ids.map(&:to_i)
+          ds = ds.where(
+            Sequel.or(
+              [
+                [ :context_specialty_concept_id, spec_con_ids ],
+                [ :practitioner_specialty_concept_id, spec_con_ids ]
+              ]
+            )
+          )
+        end
+
+        unless role_type_concept_ids.include?("*")
+          role_con_ids = role_type_concept_ids.map(&:to_i)
+          ds = ds.where(role_type_concept_id: role_con_ids)
+        end
+
+        ds
+      end
+
+      def query_columns
+        default_columns + %i[
+          context_specialty_concept_id
+          practitioner_specialty_concept_id
+          role_type_concept_id
+          provider_id
+        ]
+      end
+    end
+  end
+end

data/lib/conceptql/operators/race.rb

@@ -1,4 +1,7 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
+require "yaml"
 
 module ConceptQL
   module Operators
@@ -9,14 +12,59 @@ module ConceptQL
     # concept_name column of the concept table.  If you misspell the race name
     # you won't get any matches
     class Race < Operator
-      desc 'Finds all people that match the races'
-      argument :races, type: :codelist, vocab: 'Race'
-      types :person
+      register __FILE__
+
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      desc "Selects person records that match the given set of Race codes."
+      argument :races, type: :codelist, vocab: "Race"
+      domains :person
+      category "Select by Property"
+      basic_type :selection
+      validate_no_upstreams
+      validate_at_least_one_argument
+
+      def race_descendents
+        @race_descendents ||= YAML.load_file(ConceptQL.race_file)
+      end
+
+      def query_cols
+        table_columns(table)
+      end
+
+      def table
+        source_table
+      end
+
+      def source_table
+        if gdm?
+          :patients
+        else
+          :person
+        end
+      end
 
       def query(db)
-        db.from(:person___p)
-          .join(:vocabulary__concept___vc, { vc__concept_id: :p__race_concept_id })
-          .where(Sequel.function(:lower, :vc__concept_name) => arguments.map(&:downcase))
+        words = arguments - actual_ids
+
+        concept_ids = dm.concept_ids_by_name(db, words)
+
+        c_ids = (concept_ids + actual_ids).map { |i| [ i, race_descendents[i] ] }.flatten.compact.uniq.sort
+
+        c_ids = dm.related_concept_ids(db, c_ids) if gdm?
+
+        q = db.from(source_table)
+        if words.any? { |w| w.match(/unknown/i) }
+          c_ids += [ 0 ]
+          q = q.where(race_concept_id: nil)
+        end
+
+        q.where(race_concept_id: c_ids)
+      end
+
+      def actual_ids
+        @actual_ids ||= arguments.select { |i| i.to_s.match(/^\d+$/) }
       end
     end
   end

data/lib/conceptql/operators/read.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "sequel"
+require "sequel/adapters/mock"
+require_relative "../behaviors/labish"
+
+module ConceptQL
+  module Operators
+    class Read < Operator
+      include ConceptQL::Behaviors::CodeLister
+      register __FILE__
+
+      preferred_name "READ"
+      desc "Selects records based on the Read vocabulary."
+      argument :read_codes, type: :codelist, vocab: "Read"
+      basic_type :selection
+      category "Select by Clinical Codes"
+      conceptql_spec_id "vocabulary"
+
+      def query(db)
+        gdm? ? gdm(db) : omopv4(db)
+      end
+
+      def gdm(db)
+        ops(db).first.evaluate(db)
+      end
+
+      def omopv4(db)
+        streams = ops(db).map { |op| op.evaluate(db) }
+
+        streams.inject { |q, query| q.union(query, all: true) }.from_self
+      end
+
+      def domains(db)
+        ops.first.domains(db)
+      end
+
+      def query_cols
+        gdm? ? ops.first.query_cols : ops.map(&:query_cols).flatten.uniq
+      end
+
+      def required_columns
+        ops.flat_map(&:required_columns).uniq
+      end
+
+      private
+
+      def ops(_db = nil)
+        [ ReadGDM.new(nodifier, "read_condition_occurrence", *arguments) ]
+      end
+
+      def describe_codes(db, codes)
+        ops.flat_map { |op| op.describe_codes(db, codes) }
+      end
+
+      def klasses
+        @klasses ||= {
+          condition_occurrence: ReadCondition,
+          procedure_occurrence: ReadProcedure,
+          drug_exposure: ReadDrug,
+          observation: ReadObservation
+        }
+      end
+
+      class ReadBase < ConceptQL::Operators::Vocabulary
+        preferred_name "READ"
+        argument :read_codes, type: :codelist, vocab: "Read"
+      end
+
+      class ReadOmopBase < ReadBase
+        include ConceptQL::Vocabularies::Behaviors::Omopish
+
+        def vocabulary_id
+          17
+        end
+
+        def source_table
+          table
+        end
+
+        def domain
+          table
+        end
+
+        def domains(_db)
+          Array(domain)
+        end
+      end
+
+      class ReadGDM < ReadBase
+        include ConceptQL::Vocabularies::Behaviors::Gdmish
+        include ConceptQL::Behaviors::Labish
+
+        def vocabulary_id
+          "Read"
+        end
+
+        def domain_map(_v_id)
+          :condition_occurrence
+        end
+
+        def domains(_db)
+          [ :condition_occurrence ]
+        end
+      end
+
+      class ReadCondition < ReadOmopBase
+        def table
+          :procedure_occurrence
+        end
+
+        def source_column
+          :procedure_source_value
+        end
+
+        def concept_column
+          :procedure_concept_id
+        end
+      end
+
+      class ReadProcedure < ReadOmopBase
+        def table
+          :procedure_occurrence
+        end
+
+        def source_column
+          :procedure_source_value
+        end
+
+        def concept_column
+          :procedure_concept_id
+        end
+      end
+
+      class ReadObservation < ReadOmopBase
+        include ConceptQL::Behaviors::Labish
+
+        def table
+          :observation
+        end
+
+        def source_column
+          :observation_source_value
+        end
+
+        def concept_column
+          :observation_concept_id
+        end
+      end
+
+      class ReadDrug < ReadOmopBase
+        include ConceptQL::Behaviors::Drugish
+
+        def table
+          :drug_exposure
+        end
+
+        def source_column
+          :drug_source_value
+        end
+
+        def concept_column
+          :drug_concept_id
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/operators/recall.rb

@@ -1,4 +1,6 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
@@ -8,31 +10,64 @@ module ConceptQL
     # that complex concept into a set of sub-concepts to better understand it.
     #
     # This operator will look for a sub-concept that has been created through the
-    # "define" operator and will fetch the results cached in the corresponding table
+    # "define" operator and will fetch the records cached in the corresponding table
     class Recall < Operator
-      desc <<-EOF
-Recalls a set of named results that were previously stored using the Define operator.
-Must be surrounded by the same Let operator as surrounds the corresponding Define operator.
-      EOF
+      register __FILE__
+
+      desc "Recalls a set of records from a labeled operator in the statement"
+
       argument :name, type: :string
-      category 'Variable Assignment'
+      category "Get Related Data"
+      basic_type :selection
+      validate_no_upstreams
+      validate_one_argument
+      default_query_columns
 
       def query(db)
         scope.from(db, source)
       end
 
-      def columns(query, local_type)
-        COLUMNS
-      end
-
-      def types
-        scope.types(source)
+      def domains(db)
+        scope.domains(source, db)
       end
 
       def source
         arguments.first
       end
+
+      def annotate(db, opts = {})
+        @annotate ||= if valid?(db) && replaced?
+                        original.annotate(db, opts)
+        else
+                        super
+        end
+      end
+
+      def original
+        nodifier.scope.fetch_operator(source)
+      end
+
+      def code_list(db)
+        original.code_list(db)
+      end
+
+      private
+
+      def additional_validation(_db, _opts = {})
+        return unless arguments.length == 1
+
+        if scope.fetch_operator(source)
+          scope.recall_dependencies[source].each do |d|
+            add_error("mutually referential recalls", d) if scope.recall_dependencies[d].include?(source)
+          end
+        else
+          add_error("no matching label", source)
+        end
+      end
+
+      def replaced?
+        options[:replaced]
+      end
     end
   end
 end
-

data/lib/conceptql/operators/snf.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Snf < Operator
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
+      include ConceptQL::Behaviors::Utilizable
+
+      desc "Selects admission records of type 'SNF'"
+
+      def collection_type
+        "SNF"
+      end
+    end
+  end
+end

data/lib/conceptql/operators/sum.rb

@@ -1,29 +1,46 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Sum < PassThru
-      desc <<-EOF
-Sums value_as_number across all results that match on all but start_date, end_date.
-For start_date and end_date the min and max of each respectively is returned.'
+      register __FILE__
+
+      def events_per_patient
+        :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc <<~EOF
+        Sums value_as_number across all records that match on all but start_date, end_date.
+        For start_date and end_date the min and max of each respectively is returned.'
       EOF
+      default_query_columns
+      validate_at_least_one_upstream
+      validate_no_arguments
+      require_column :value_as_number
 
       def query(db)
         db.from(unioned(db))
-          .select_group(*(COLUMNS - [:start_date, :end_date, :criterion_id, :value_as_number]))
-          .select_append(Sequel.lit('?', 0).as(:criterion_id))
-          .select_append{ min(start_date).as(:start_date) }
-          .select_append{ max(end_date).as(:end_date) }
-          .select_append{sum(value_as_number).as(:value_as_number)}
+          .select_group(*(dynamic_columns - %i[start_date end_date criterion_id value_as_number]))
+          .select_append(Sequel.lit("?", 0).as(:criterion_id))
+          .select_append { min(start_date).as(:start_date) }
+          .select_append { max(end_date).as(:end_date) }
+          .select_append { sum(value_as_number).as(:value_as_number) }
           .from_self
       end
 
       def unioned(db)
-        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
-          uni.union(q)
+        upstreams.map { |c| c.evaluate(db).select(*query_cols) }.inject do |uni, q|
+          uni.union(q.from_self)
         end
       end
     end
   end
 end
-