conceptql 0.3.0 → 0.4.1

data/lib/conceptql/operators/pass_thru.rb

@@ -1,20 +1,24 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
+    # The base class of operators whose domains and columns are those of their
+    # upstreams (union, count, sum, numeric, dedup, episode, time_window and
+    # co_reported). It has no query method, so it does not register: pass_thru
+    # is not an operator name.
     class PassThru < Operator
-      register __FILE__
-
       basic_type :set
       no_desc
 
       def domains(db)
         doms = upstreams.compact.flat_map { |up| up.domains(db) }.uniq
-        doms.empty? ? [:invalid] : doms
+        doms.empty? ? [ :invalid ] : doms
       end
 
       def query_cols
-        upstreams.first.query_cols
+        upstreams.map(&:query_cols).inject(:|)
       end
     end
   end

data/lib/conceptql/operators/person.rb

@@ -1,11 +1,17 @@
-require_relative 'casting_operator'
+# frozen_string_literal: true
+
+require_relative "casting_operator"
 
 module ConceptQL
   module Operators
     class Person < CastingOperator
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
       register __FILE__
 
-      desc 'Generates all person records, or, if fed a stream, fetches all person records for the people represented in the incoming result set.'
+      desc "Selects all records in the person table."
+
       allows_one_upstream
       domains :person
 
@@ -13,6 +19,10 @@ module ConceptQL
         :person
       end
 
+      def source_table
+        dm.table_by_domain(my_domain)
+      end
+
       def i_point_at
         []
       end

data/lib/conceptql/operators/person_filter.rb

@@ -1,16 +1,18 @@
-require_relative 'binary_operator_operator'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
 
 module ConceptQL
   module Operators
     class PersonFilter < BinaryOperatorOperator
       register __FILE__
 
-      desc 'If a result in the left hand results (LHR) matches a person in the right hand results (RHR), it is passed through.'
+      desc "Passes along left hand records with a corresponding person_id in the right hand records."
       default_query_columns
 
       def query(db)
         db.from(left.evaluate(db))
-          .where(person_id: right.evaluate(db).select_group(:person_id))
+          .where(person_id: right.evaluate(db).from_self.select_group(:person_id))
       end
     end
   end

data/lib/conceptql/operators/place_of_service_filter.rb

@@ -1,46 +1,42 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
-    # Filters the incoming stream of events to only those that have a
+    # Filters the incoming stream of records to only those that have a
     # an associated visit_occurrence with the matching place_of_service
     #
     # Provenance related concepts are the ones found in the xxx_type_concept_id
     # field.
     #
-    # If the event has NULL for the provenance-related field, they are filtered
+    # If the record has NULL for the provenance-related field, they are filtered
     # out.
     #
     # Multiple provenances can be specified at once
     class PlaceOfServiceFilter < Operator
       register __FILE__
 
-      desc <<-EOF
-Filters records to include only those that match one or more of the Medicare Place Of Service values.
+      desc "Passes along records that match one or more of the Medicare Place Of Service values."
 
-Common values include 21 (inpatient hospital), 23 (emergency room), and 11 (office).
-      EOF
-      argument :places_of_service, type: :codelist, vocab: 'Place of Service'
+      argument :places_of_service, type: :codelist, vocab: "Place of Service"
       category "Filter Single Stream"
       basic_type :temporal
       allows_one_upstream
       validate_one_upstream
       validate_at_least_one_argument
-      require_column :place_of_service_concept_id
+      require_column :visit_source_concept_id
       default_query_columns
 
       def query(db)
         db.from(stream.evaluate(db))
-          .where(place_of_service_concept_id: place_of_service_concept_ids(db))
+          .where(visit_source_concept_id: place_of_service_concept_ids(db))
       end
 
-    private
+      private
 
       def place_of_service_concept_ids(db)
-        db.from(:concept)
-          .where(concept_code: arguments.map(&:to_s))
-          .where(vocabulary_id: 14)
-          .select(:concept_id)
+        dm.concept_ids(db, "Place of Service", arguments.map(&:to_s))
       end
     end
   end

data/lib/conceptql/operators/provenance.rb

@@ -1,66 +1,55 @@
-require_relative 'operator'
-require_relative '../behaviors/provenanceable'
-require 'facets/kernel/present'
+# frozen_string_literal: true
+
+require_relative "operator"
+require_relative "../behaviors/provenanceable"
 
 module ConceptQL
   module Operators
-    # Filters the incoming stream of events to only those that have a
+    # Filters the incoming stream of records to only those that have a
     # provenance-related concept_id.
     #
     # Provenance related concepts are the ones found in the xxx_type_concept_id
     # field.
     #
-    # If the event has NULL for the provenance-related field, they are filtered
+    # If the record has NULL for the provenance-related field, they are filtered
     # out.
     #
     # Multiple provenances can be specified at once
     class Provenance < Operator
-      register __FILE__
-
-      include ConceptQL::Provenanceable
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
 
-      desc <<-EOF
-Filters incoming events to those with the indicated provenance.
+      desc "Passes along records with the indicated provenance (e.g. inpatient, outpatient, file type)."
 
-Enter numeric concept id(s), or the corresponding text label(s)
-(e.g., "inpatient", "outpatient", "carrier_claim").
-      EOF
-      argument :provenance_types, label: 'Provenance Types', type: :codelist
+      argument :provenance_types, label: "Provenance Types", type: :codelist
       category "Filter Single Stream"
       basic_type :temporal
       allows_one_upstream
       validate_one_upstream
-      require_column :provenance_type
       default_query_columns
 
+      include ConceptQL::Provenanceable
+
+      attr_reader :db
+
       def query(db)
-        db.from(stream.evaluate(db))
-          .where(provenance_type: provenance_concept_ids)
+        @db = db
+        db.from(stream.evaluate(db)).where(build_where_from_codes(db, arguments))
       end
 
-    private
+      private
 
-      def validate(db, opts = {})
-        super
-        bad_keywords = all_args.select { |arg| arg.to_i.zero? }
-                        .reject { |arg| concept_ids.keys.include?(arg.to_sym) }
+      def additional_validation(db, _opts = {})
+        # With no lexicon there are no provenance concepts to check against,
+        # and every keyword would be reported as unrecognized. Vocabulary
+        # operators skip their unknown-code check in the same situation.
+        return if dm.lexicon.strategy == :no_db
 
-        if bad_keywords.present?
-          add_error("unrecognized keywords", *bad_keywords)
-        end
-      end
-
-      def provenance_concept_ids
-        all_args.flat_map do |arg|
-          to_concept_id(arg)
-        end
-      end
+        bad_keywords = find_bad_keywords(db, arguments)
+        return unless bad_keywords.present?
 
-      def all_args
-        arguments.map(&:to_s).flat_map { |w| w.split(/\s*,\s*/) }.uniq
+        add_error("unrecognized keywords", *bad_keywords.uniq)
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/provider_filter.rb

@@ -1,36 +1,62 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
-    # Filters incoming events to only those that have been associated with
+    # Filters incoming records to only those that have been associated with
     # providers matching the given criteria.
     class ProviderFilter < Operator
       register __FILE__
 
-      desc "Filters incoming events to only those that match the associated providers based on provider specialty concept_ids."
+      desc "Passes along records where the provider's specialty matches a given set of specialty_concept_ids.\n\nA single record may have multiple specialties, so the operator will return all records that match any of the given specialties, which may lead to duplicate records."
       option :specialties, type: :string
+      option :roles, type: :string
       category "Filter Single Stream"
       basic_type :temporal
       allows_one_upstream
       validate_one_upstream
-      validate_required_options :specialties
+      validate_required_options :specialties, :roles
       require_column :provider_id
+      require_column :context_specialty_concept_id
+      require_column :practitioner_specialty_concept_id
+      require_column :role_type_concept_id
       default_query_columns
 
       def query(db)
-        db.from(stream.evaluate(db))
-          .where(provider_id: matching_provider_ids(db))
+        ds = stream.evaluate(db).from_self(alias: :upstream)
+
+        specialty_concept_ids = options[:specialties].split(/\s*,\s*/)
+        role_type_concept_ids = options[:roles].split(/\s*,\s*/)
+
+        unless specialty_concept_ids.include?("*")
+          spec_con_ids = specialty_concept_ids.map(&:to_i)
+          ds = ds.where(
+            Sequel.or(
+              [
+                [ :context_specialty_concept_id, spec_con_ids ],
+                [ :practitioner_specialty_concept_id, spec_con_ids ]
+              ]
+            )
+          )
+        end
+
+        unless role_type_concept_ids.include?("*")
+          role_con_ids = role_type_concept_ids.map(&:to_i)
+          ds = ds.where(role_type_concept_id: role_con_ids)
+        end
+
+        ds
       end
 
-    private
-      def matching_provider_ids(db)
-        specialty_concept_ids = options[:specialties].split(/\s*,\s*/).map(&:to_i)
-        db.from(:provider)
-          .where(specialty_concept_id: specialty_concept_ids)
-          .select(:provider_id)
+      def query_columns
+        default_columns + %i[
+          context_specialty_concept_id
+          practitioner_specialty_concept_id
+          role_type_concept_id
+          provider_id
+        ]
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/race.rb

@@ -1,4 +1,7 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
+require "yaml"
 
 module ConceptQL
   module Operators
@@ -11,19 +14,57 @@ module ConceptQL
     class Race < Operator
       register __FILE__
 
-      desc 'Generates all person records that match the given set of Race codes.'
-      argument :races, type: :codelist, vocab: 'Race'
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      desc "Selects person records that match the given set of Race codes."
+      argument :races, type: :codelist, vocab: "Race"
       domains :person
       category "Select by Property"
       basic_type :selection
-      query_columns :person, :concept
       validate_no_upstreams
       validate_at_least_one_argument
 
+      def race_descendents
+        @race_descendents ||= YAML.load_file(ConceptQL.race_file)
+      end
+
+      def query_cols
+        table_columns(table)
+      end
+
+      def table
+        source_table
+      end
+
+      def source_table
+        if gdm?
+          :patients
+        else
+          :person
+        end
+      end
+
       def query(db)
-        db.from(:person___p)
-          .join(:concept___c, { c__concept_id: :p__race_concept_id })
-          .where(Sequel.function(:lower, :c__concept_name) => arguments.map(&:downcase))
+        words = arguments - actual_ids
+
+        concept_ids = dm.concept_ids_by_name(db, words)
+
+        c_ids = (concept_ids + actual_ids).map { |i| [ i, race_descendents[i] ] }.flatten.compact.uniq.sort
+
+        c_ids = dm.related_concept_ids(db, c_ids) if gdm?
+
+        q = db.from(source_table)
+        if words.any? { |w| w.match(/unknown/i) }
+          c_ids += [ 0 ]
+          q = q.where(race_concept_id: nil)
+        end
+
+        q.where(race_concept_id: c_ids)
+      end
+
+      def actual_ids
+        @actual_ids ||= arguments.select { |i| i.to_s.match(/^\d+$/) }
       end
     end
   end

data/lib/conceptql/operators/read.rb

@@ -1,67 +1,56 @@
-require 'sequel'
-require 'sequel/adapters/mock'
+# frozen_string_literal: true
+
+require "sequel"
+require "sequel/adapters/mock"
+require_relative "../behaviors/labish"
 
 module ConceptQL
   module Operators
     class Read < Operator
+      include ConceptQL::Behaviors::CodeLister
       register __FILE__
 
       preferred_name "READ"
-      desc "Selects results from the the condition_occurrence, procedure_occurrence, drug_exposure, and observation tables based on the READ codes provided."
+      desc "Selects records based on the Read vocabulary."
       argument :read_codes, type: :codelist, vocab: "Read"
       basic_type :selection
       category "Select by Clinical Codes"
-      default_query_columns
+      conceptql_spec_id "vocabulary"
 
       def query(db)
-        ops = codes_by_domain(db).map do |domain, codes|
-          klasses[domain].new(self.nodifier, *codes)
-        end
+        gdm? ? gdm(db) : omopv4(db)
+      end
+
+      def gdm(db)
+        ops(db).first.evaluate(db)
+      end
 
-        streams = ops.map { |op| op.evaluate(db) }
+      def omopv4(db)
+        streams = ops(db).map { |op| op.evaluate(db) }
 
         streams.inject { |q, query| q.union(query, all: true) }.from_self
       end
 
       def domains(db)
-        codes_by_domain(db).keys
+        ops.first.domains(db)
       end
 
-      private
-
-      def codes_by_domain(db)
-        @codes_by_domain ||= get_codes_by_domain(db)
+      def query_cols
+        gdm? ? ops.first.query_cols : ops.map(&:query_cols).flatten.uniq
       end
 
-      def get_codes_by_domain(db)
-        return { observation: arguments } if db.is_a?(::Sequel::Mock)
-        codes_and_mapping_types = db[:source_to_concept_map]
-          .where(source_code: arguments, source_vocabulary_id: 17)
-          .select_map([:source_code, :mapping_type])
+      def required_columns
+        ops.flat_map(&:required_columns).uniq
+      end
 
-        doms_and_codes = codes_and_mapping_types.group_by(&:last).each_with_object({}) do |(mapping_type, codes_and_maps), doms|
-          dom = mapping_type_to_domain(mapping_type)
-          doms[dom] ||= []
-          doms[dom] += codes_and_maps.map(&:first)
-        end
+      private
 
-        leftovers = arguments - doms_and_codes.flat_map { |k, v| v }
-        doms_and_codes[:observation] ||= []
-        doms_and_codes[:observation] += leftovers
-        doms_and_codes
+      def ops(_db = nil)
+        [ ReadGDM.new(nodifier, "read_condition_occurrence", *arguments) ]
       end
 
-      def mapping_type_to_domain(mapping_type)
-        case mapping_type.downcase.to_sym
-        when :condition
-          :condition_occurrence
-        when :procedure
-          :procedure_occurrence
-        when :drug
-          :drug_exposure
-        else
-          :observation
-        end
+      def describe_codes(db, codes)
+        ops.flat_map { |op| op.describe_codes(db, codes) }
       end
 
       def klasses
@@ -73,25 +62,65 @@ module ConceptQL
         }
       end
 
-      class ReadCondition < ConditionOccurrenceSourceVocabularyOperator
+      class ReadBase < ConceptQL::Operators::Vocabulary
         preferred_name "READ"
         argument :read_codes, type: :codelist, vocab: "Read"
+      end
+
+      class ReadOmopBase < ReadBase
+        include ConceptQL::Vocabularies::Behaviors::Omopish
 
         def vocabulary_id
           17
         end
+
+        def source_table
+          table
+        end
+
+        def domain
+          table
+        end
+
+        def domains(_db)
+          Array(domain)
+        end
       end
 
-      class ReadProcedure < SourceVocabularyOperator
-        preferred_name "READ"
-        argument :read_codes, type: :codelist, vocab: "Read"
+      class ReadGDM < ReadBase
+        include ConceptQL::Vocabularies::Behaviors::Gdmish
+        include ConceptQL::Behaviors::Labish
 
+        def vocabulary_id
+          "Read"
+        end
+
+        def domain_map(_v_id)
+          :condition_occurrence
+        end
+
+        def domains(_db)
+          [ :condition_occurrence ]
+        end
+      end
+
+      class ReadCondition < ReadOmopBase
         def table
           :procedure_occurrence
         end
 
-        def vocabulary_id
-          17
+        def source_column
+          :procedure_source_value
+        end
+
+        def concept_column
+          :procedure_concept_id
+        end
+      end
+
+      class ReadProcedure < ReadOmopBase
+        def table
+          :procedure_occurrence
         end
 
         def source_column
@@ -103,18 +132,13 @@ module ConceptQL
         end
       end
 
-      class ReadObservation < SourceVocabularyOperator
-        preferred_name "READ"
-        argument :read_codes, type: :codelist, vocab: "Read"
+      class ReadObservation < ReadOmopBase
+        include ConceptQL::Behaviors::Labish
 
         def table
           :observation
         end
 
-        def vocabulary_id
-          17
-        end
-
         def source_column
           :observation_source_value
         end
@@ -124,18 +148,13 @@ module ConceptQL
         end
       end
 
-      class ReadDrug < SourceVocabularyOperator
-        preferred_name "READ"
-        argument :read_codes, type: :codelist, vocab: "Read"
+      class ReadDrug < ReadOmopBase
+        include ConceptQL::Behaviors::Drugish
 
         def table
           :drug_exposure
         end
 
-        def vocabulary_id
-          17
-        end
-
         def source_column
           :drug_source_value
         end

data/lib/conceptql/operators/recall.rb

@@ -1,4 +1,6 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
@@ -8,14 +10,12 @@ module ConceptQL
     # that complex concept into a set of sub-concepts to better understand it.
     #
     # This operator will look for a sub-concept that has been created through the
-    # "define" operator and will fetch the results cached in the corresponding table
+    # "define" operator and will fetch the records cached in the corresponding table
     class Recall < Operator
       register __FILE__
 
-      desc <<-EOF
-Recalls a set of named results that were previously stored using the Define operator.
-Must be surrounded by the same Let operator as surrounds the corresponding Define operator.
-      EOF
+      desc "Recalls a set of records from a labeled operator in the statement"
+
       argument :name, type: :string
       category "Get Related Data"
       basic_type :selection
@@ -37,9 +37,9 @@ Must be surrounded by the same Let operator as surrounds the corresponding Defin
 
       def annotate(db, opts = {})
         @annotate ||= if valid?(db) && replaced?
-          original.annotate(db, opts)
+                        original.annotate(db, opts)
         else
-          super
+                        super
         end
       end
 
@@ -47,20 +47,21 @@ Must be surrounded by the same Let operator as surrounds the corresponding Defin
         nodifier.scope.fetch_operator(source)
       end
 
+      def code_list(db)
+        original.code_list(db)
+      end
+
       private
 
-      def validate(db, opts = {})
-        super
-        if arguments.length == 1
-          if scope.fetch_operator(source)
-            scope.recall_dependencies[source].each do |d|
-              if scope.recall_dependencies[d].include?(source)
-                add_error("mutually referential recalls", d)
-              end
-            end
-          else
-            add_error("no matching label", source)
+      def additional_validation(_db, _opts = {})
+        return unless arguments.length == 1
+
+        if scope.fetch_operator(source)
+          scope.recall_dependencies[source].each do |d|
+            add_error("mutually referential recalls", d) if scope.recall_dependencies[d].include?(source)
           end
+        else
+          add_error("no matching label", source)
         end
       end
 
@@ -70,4 +71,3 @@ Must be surrounded by the same Let operator as surrounds the corresponding Defin
     end
   end
 end
-