conceptql 0.2.0 → 0.4.0

data/lib/conceptql/operators/casting_operator.rb

@@ -1,35 +1,41 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
     # Parent class of all casting operators
     #
     # Subclasses must implement the following methods:
-    # - my_type
+    # - my_domain
     # - i_point_at
     # - these_point_at_me
     #
-    # i_point_at returns a list of types for which the operator's table of origin
+    # i_point_at returns a list of domains for which the operator's table of origin
     # has foreign_keys pointing to another table, e.g.:
     # procedure_occurrence has an FK to visit_occurrence, so we'd put
     # :visit_occurrence in the i_point_at array
     #
-    # these_point_at_me is a list of types for which that type's table
+    # these_point_at_me is a list of domains for which that domain's table
     # of origin has a FK  pointing to the current operator's
     # table of origin, e.g.:
     # procedure_cost has an FK to procedure_occurrence so we'd
     # put :procedure_cost in procedure_occurrence's these_point_at_me array
     #
     # Also, if a casting operator is passed no streams, it will return all the
-    # rows in its table as results.
+    # rows in its table as records.
     class CastingOperator < Operator
-      category 'Casting'
-      def types
-        [type]
+      category "Get Related Data"
+      basic_type :cast
+      validate_at_most_one_upstream
+      validate_no_arguments
+
+      def domains(_db)
+        [ domain ]
       end
 
-      def type
-        my_type
+      def domain
+        my_domain
       end
 
       def castables
@@ -37,57 +43,66 @@ module ConceptQL
       end
 
       def query(db)
-        return db.from(make_table_name(my_type)) if stream.nil?
+        return db.from(table) if stream.nil?
+
         base_query(db, stream.evaluate(db))
       end
 
+      def table
+        dm.table_by_domain(my_domain)
+      end
+
+      def make_table_name(domain)
+        super(dm.table_by_domain(domain))
+      end
+
       private
 
       def base_query(db, stream_query)
-        uncastable_types = stream.types - castables
-        to_me_types = stream.types & these_point_at_me
-        from_me_types = stream.types & i_point_at
+        uncastable_domains = stream.domains(db) - castables
+        to_me_domains = stream.domains(db) & these_point_at_me
+        from_me_domains = stream.domains(db) & i_point_at
 
-        destination_table = make_table_name(my_type)
+        destination_table = make_table_name(my_domain)
         casting_query = db.from(destination_table)
         wheres = []
 
-        unless uncastable_types.empty?
+        unless uncastable_domains.empty?
           # We have a situation where one or more of the incoming streams
           # isn't castable so we'll just return all rows for
           # all people in each uncastable stream
           uncastable_person_ids = db.from(stream_query)
-            .where(criterion_type: uncastable_types.map(&:to_s))
-            .select_group(:person_id)
-          wheres << Sequel.expr(person_id: uncastable_person_ids)
+                                    .where(criterion_domain: uncastable_domains.map(&:to_s))
+                                    .select_group(:person_id)
+          wheres << Sequel.expr(dm.person_id(table) => uncastable_person_ids)
         end
 
-        destination_type_id = make_type_id(my_type)
+        destination_domain_id = dm.make_table_id(my_domain)
 
-        unless to_me_types.empty?
-          # For each castable type in the stream, setup a query that
-          # casts each type to a set of IDs, union those IDs and fetch
+        unless to_me_domains.empty?
+          # For each castable domain in the stream, setup a query that
+          # casts each domain to a set of IDs, union those IDs and fetch
           # them from the source table
-          castable_type_query = to_me_types.map do |source_type|
+          castable_domain_query = to_me_domains.map do |source_domain|
             source_ids = db.from(stream_query)
-              .where(criterion_type: source_type.to_s)
-              .select_group(:criterion_id)
-            source_table = make_table_name(source_type)
-            source_type_id = make_type_id(source_type)
+                           .where(criterion_domain: source_domain.to_s)
+                           .select_group(:criterion_id)
+            source_table = make_table_name(source_domain)
+            source_domain_id = dm.make_fk_id(source_domain)
 
             db.from(source_table)
-              .where(source_type_id => source_ids)
-              .select(destination_type_id)
+              .where(source_domain_id => source_ids)
+              .select(destination_domain_id)
           end.inject do |union_query, q|
-            union_query.union(q)
+            union_query.union(q, all: true)
           end
-          wheres << Sequel.expr(destination_type_id => castable_type_query)
+          wheres << Sequel.expr(destination_domain_id => castable_domain_query)
         end
 
-        unless from_me_types.empty?
-          from_me_types.each do |from_me_type|
-            fk_type_id = make_type_id(from_me_type)
-            wheres << Sequel.expr(fk_type_id => db.from(stream_query).where(criterion_type: from_me_type.to_s).select_group(:criterion_id))
+        unless from_me_domains.empty?
+          from_me_domains.each do |from_me_domain|
+            fk_domain_id = dm.fk_by_domain(from_me_domain)
+            wheres << Sequel.expr(fk_domain_id => db.from(stream_query).where(criterion_domain: from_me_domain.to_s).select_group(:criterion_id))
           end
         end
 

data/lib/conceptql/operators/co_reported.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+
+module ConceptQL
+  module Operators
+    class CoReported < PassThru
+      register __FILE__
+
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Passes along all events that were co-reported in the same record in the source data."
+
+      allows_many_upstreams
+      category "Combine Streams"
+      default_query_columns
+      validate_at_least_one_upstream
+      validate_no_arguments
+      require_column :context_id
+
+      def query(db)
+        if gdm?
+          gdm(db)
+        else
+          events_with_common_visits(db).inject { |q, query| q.union(query, all: true) }
+        end
+      end
+
+      def gdm(db)
+        contexteds = upstreams.map do |stream|
+          [ stream.cte_name(:contextified), contextify(db, stream) ]
+        end.to_h
+
+        # Get all context_id's that are in all streams and do not share the same critierion_id
+        first, *rest = *contexteds.keys
+        shared_context_ids = rest.inject(db[first].select(:context_id)) do |q, next_cte|
+          q.intersect(db[next_cte].select(:context_id))
+        end
+        # shared_context_ids = rest.inject(first.from_self(alias: :first)) do |q, shared_event|
+        #    q.join(shared_event.select(:context_id, :criterion_id), context_id: :context_id) do |a,b|
+        #    ~(Sequel.qualify(a,:criterion_id) =~ Sequel.qualify(b,:criterion_id) )
+        #  end.select(Sequel[:first][:context_id]).distinct
+        # end.from_self
+
+        if ConceptQL.avoid_ctes?
+          context_id_ds = shared_context_ids
+        else
+          name = cte_name(:shared_context_ids)
+          context_id_ds = db[name]
+        end
+
+        shared_events = contexteds.keys.map do |contexted|
+          db[contexted].where(context_id: context_id_ds).select(*query_cols)
+        end
+
+        ds = shared_events.inject do |q, shared_event|
+          q.union(shared_event)
+        end
+
+        contexteds.each do |name, query|
+          ds = ds.with(name, query)
+        end
+        if name
+          opts = {}
+          opts[:materialized] = true if rdbms.supports_materialized?
+          ds = ds.with(name, shared_context_ids, opts)
+        end
+
+        ds
+      end
+
+      def contextify(db, stream)
+        stream.evaluate(db).from_self(alias: :s)
+        # .join(Sequel[:clinical_codes].as(:c), id: :criterion_id)
+        # .select_all(:s)
+        # .select_append(Sequel[:c][:context_id].as(:context_id))
+        # .from_self
+      end
+
+      private
+
+      def visit_occurrence_ids_in_common(db)
+        @visit_occurrence_ids_in_common ||= upstream_queries(db).map do |q|
+          q.select(:visit_occurrence_id)
+        end.inject do |q, query|
+          q.from_self(alias: :tab1)
+           .join(query.as(:tab2), visit_occurrence_id: :visit_occurrence_id)
+           .select(Sequel[:tab1][:visit_occurrence_id].as(:visit_occurrence_id))
+        end
+      end
+
+      def events_with_common_visits(db)
+        upstream_queries(db).map { |q| q.where(visit_occurrence_id: visit_occurrence_ids_in_common(db)).from_self }
+      end
+
+      def upstream_queries(db)
+        @upstream_queries ||= upstreams.map do |expression|
+          expression.evaluate(db).select(*query_cols).from_self
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/operators/concurrent_within.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+require_relative "../date_adjuster"
+
+module ConceptQL
+  module Operators
+    class ConcurrentWithin < Operator
+      register __FILE__
+
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Filters each upstream to only include records where other upstreams' date ranges overlap"
+      option :start, type: :string
+      option :end, type: :string
+      allows_many_upstreams
+      validate_at_least_one_upstream
+      validate_no_arguments
+      validate_option DateAdjuster::VALID_INPUT, :start, :end
+      category "Combine Streams"
+      basic_type :set
+      default_query_columns
+
+      def query(db)
+        datasets = upstreams.map do |stream|
+          stream.evaluate(db)
+        end
+
+        return datasets.first.from_self if datasets.length == 1
+
+        adjusted_start_date = DateAdjuster.new(self, options[:start]).adjust(Sequel[:l][:start_date], true)
+        adjusted_end_date = DateAdjuster.new(self, options[:end]).adjust(Sequel[:l][:end_date])
+
+        datasets = datasets.map do |ds|
+          matching = ds.from_self(alias: :l)
+
+          (datasets - [ ds ]).each do |other|
+            other = other
+                    .from_self(alias: :r)
+                    .where(adjusted_start_date <= Sequel[:r][:start_date])
+                    .where(adjusted_end_date >= Sequel[:r][:end_date])
+                    .select(*matching_columns)
+
+            matching = matching.where(other.where(matching_columns.map do |x|
+              [ Sequel.qualify(:l, x), Sequel.qualify(:r, x) ]
+            end).exists)
+          end
+
+          matching
+        end
+
+        ds, *rest = datasets
+        rest.each do |other|
+          ds = ds.union(other, from_self: nil)
+        end
+
+        ds.from_self
+      end
+    end
+  end
+end

data/lib/conceptql/operators/contains.rb

@@ -1,16 +1,20 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class Contains < TemporalOperator
-      desc <<-EOF
-Any result in the LHR whose start_date is on or before and whose end_date is on or after a result from the RHR.
-L--X-L
-R-----R
-L------Y--------L
-EOF
+      register __FILE__
+
+      desc "For each person, passes along left hand records with a start_date and end_date containing a right hand record's start_date and end_date."
+
       def where_clause
-        [Proc.new { l__start_date <= r__start_date}, Proc.new { r__end_date <= l__end_date }]
+        (within_start <= r_start_date) & (r_end_date <= within_end)
+      end
+
+      def within_source_table
+        Sequel[:l]
       end
     end
   end

data/lib/conceptql/operators/count.rb

@@ -1,26 +1,34 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Count < PassThru
-      desc 'Counts the number of results the exactly match across all columns.'
+      register __FILE__
+
+      desc "Counts the number of records that exactly match across all columns."
       allows_one_upstream
+      validate_one_upstream
+      validate_no_arguments
+      require_column :value_as_number
+
+      def query_cols
+        dynamic_columns - [ :value_as_number ] + [ :value_as_number ]
+      end
 
       def query(db)
         db.from(unioned(db))
-          .group(*COLUMNS)
-          .select(*(COLUMNS - [:value_as_number]))
-          .select_append{count(1).as(:value_as_number)}
+          .select_group(*(query_cols - [ :value_as_number ]))
+          .select_append { count(1).as(:value_as_number) }
           .from_self
       end
 
       def unioned(db)
-        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
-          uni.union(q)
+        upstreams.map { |c| c.evaluate(db).select(*query_cols) }.inject do |uni, q|
+          uni.union(q.from_self)
         end
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/date_range.rb

@@ -1,4 +1,7 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
+require "date"
 
 module ConceptQL
   module Operators
@@ -8,23 +11,43 @@ module ConceptQL
     # 'START' represents the first date of data in the data source,
     # 'END' represents the last date of data in the data source,
     class DateRange < Operator
-      desc 'Used to represent a date literal.'
+      register __FILE__
+
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      DATE_FORMAT = /\A#{Regexp.union([ /START/i, /END/i, /\d{4}-\d{2}-\d{2}/ ])}\z/.freeze
+
+      desc "Creates a set of all person records with the given start_date and end_date."
       option :start, type: :string
       option :end, type: :string
-      category %(Temporal Manipulation)
+      category "Select by Property"
+      basic_type :selection
+      validate_no_upstreams
+      validate_no_arguments
+      validate_option DATE_FORMAT, :start, :end
+      validate_required_options :start, :end
 
       def query(db)
-        db.from(:person)
-          .select_append(Sequel.cast_string('person').as(:criterion_type))
-          .select_append(Sequel.expr(:person_id).as(:criterion_id))
-          .select_append(start_date(db).as(:start_date), end_date(db).as(:end_date)).from_self
+        replace = {
+          start_date: start_date(db),
+          end_date: end_date(db)
+        }
+        db.from(source_table)
+          .select(*dm.columns(table: source_table, replace: replace))
+          .from_self
       end
 
-      def types
-        [:person]
+      def domains(_db)
+        [ :person ]
       end
 
       private
+
+      def source_table
+        dm.table_by_domain(:person)
+      end
+
       def start_date(db)
         date_from(db, options[:start])
       end
@@ -36,12 +59,23 @@ module ConceptQL
       # TODO: Select the earliest and latest dates of observation from
       # the proper CDM table to represent the start and end of data
       def date_from(db, str)
-        return db.from(:observation_period).select { min(:observation_period_start_date) } if str.upcase == 'START'
-        return db.from(:observation_period).select { max(:observation_period_end_date) } if str.upcase == 'END'
-        return Sequel.lit('CONVERT(DATETIME, ?)', str) if db.database_type == :mssql
-        return Sequel.lit('date ?', str)
+        return db.from(period_table).get { |o| o.min(period_start_date(db)) } if str.upcase == "START"
+        return db.from(period_table).get { |o| o.max(period_end_date(db)) } if str.upcase == "END"
+
+        str
+      end
+
+      def period_table
+        dm.period_table
+      end
+
+      def period_start_date(db)
+        dm.start_date_column(db, period_table)
+      end
+
+      def period_end_date(db)
+        dm.end_date_column(db, period_table)
       end
     end
   end
 end
-

data/lib/conceptql/operators/death.rb

@@ -1,16 +1,26 @@
-require_relative 'casting_operator'
+# frozen_string_literal: true
+
+require_relative "casting_operator"
 
 module ConceptQL
   module Operators
     class Death < CastingOperator
-      desc 'Generates all death records, or, if fed a stream, fetches all death records for the people represented in the incoming result set.'
-      types :death
+      include ConceptQL::Behaviors::Windowable
+
+      register __FILE__
+
+      desc "Selects all death records"
+      domains :death
       allows_one_upstream
 
-      def my_type
+      def my_domain
         :death
       end
 
+      def source_table
+        dm.table_by_domain(:death)
+      end
+
       def i_point_at
         [ :person ]
       end

data/lib/conceptql/operators/dedup.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+
+module ConceptQL
+  module Operators
+    class Dedup < PassThru
+      register __FILE__
+
+      desc "Counts the number of records that exactly match across all columns."
+      allows_one_upstream
+      validate_one_upstream
+
+      def query(db)
+        db.from(unioned(db))
+          .select_group(*query_cols)
+          .from_self
+      end
+
+      def query_cols
+        dynamic_columns
+      end
+
+      def unioned(db)
+        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
+          uni.union(q)
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/operators/during.rb

@@ -1,21 +1,16 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class During < TemporalOperator
-      desc <<-EOF
-Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand resuls (RHR).
-For any result in the LHR whose start_date and end_date occur within the start_date and end_date of a RHR row, that result is passed through.
-All other results are discarded, including all results in the RHR.
-      EOF
+      register __FILE__
+
+      desc "For each person, passes along left hand records with a start_date and end_date within a right hand record's start_date and end_date."
 
       def where_clause
-        if inclusive?
-          Sequel.expr(Sequel.expr(Proc.new { r__start_date <= l__start_date}).&(Sequel.expr( Proc.new { l__start_date <= r__end_date })))
-            .|(Sequel.expr(Proc.new { r__start_date <= l__end_date}).&(Sequel.expr( Proc.new { l__end_date <= r__end_date })))
-        else
-          [Proc.new { r__start_date <= l__start_date}, Proc.new { l__end_date <= r__end_date }]
-        end
+        (within_start <= l_start_date) & (l_end_date <= within_end)
       end
     end
   end