conceptql 0.3.0 → 0.4.0

data/lib/conceptql/operators/casting_operator.rb

@@ -1,4 +1,6 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
@@ -21,15 +23,15 @@ module ConceptQL
     # put :procedure_cost in procedure_occurrence's these_point_at_me array
     #
     # Also, if a casting operator is passed no streams, it will return all the
-    # rows in its table as results.
+    # rows in its table as records.
     class CastingOperator < Operator
       category "Get Related Data"
       basic_type :cast
       validate_at_most_one_upstream
       validate_no_arguments
 
-      def domains(db)
-        [domain]
+      def domains(_db)
+        [ domain ]
       end
 
       def domain
@@ -40,15 +42,20 @@ module ConceptQL
         (i_point_at + these_point_at_me)
       end
 
-      def query_cols
-        table_columns(make_table_name(my_domain))
-      end
-
       def query(db)
-        return db.from(make_table_name(my_domain)) if stream.nil?
+        return db.from(table) if stream.nil?
+
         base_query(db, stream.evaluate(db))
       end
 
+      def table
+        dm.table_by_domain(my_domain)
+      end
+
+      def make_table_name(domain)
+        super(dm.table_by_domain(domain))
+      end
+
       private
 
       def base_query(db, stream_query)
@@ -65,12 +72,12 @@ module ConceptQL
           # isn't castable so we'll just return all rows for
           # all people in each uncastable stream
           uncastable_person_ids = db.from(stream_query)
-            .where(criterion_domain: uncastable_domains.map(&:to_s))
-            .select_group(:person_id)
-          wheres << Sequel.expr(person_id: uncastable_person_ids)
+                                    .where(criterion_domain: uncastable_domains.map(&:to_s))
+                                    .select_group(:person_id)
+          wheres << Sequel.expr(dm.person_id(table) => uncastable_person_ids)
         end
 
-        destination_domain_id = make_domain_id(my_domain)
+        destination_domain_id = dm.make_table_id(my_domain)
 
         unless to_me_domains.empty?
           # For each castable domain in the stream, setup a query that
@@ -78,10 +85,10 @@ module ConceptQL
           # them from the source table
           castable_domain_query = to_me_domains.map do |source_domain|
             source_ids = db.from(stream_query)
-              .where(criterion_domain: source_domain.to_s)
-              .select_group(:criterion_id)
+                           .where(criterion_domain: source_domain.to_s)
+                           .select_group(:criterion_id)
             source_table = make_table_name(source_domain)
-            source_domain_id = make_domain_id(source_domain)
+            source_domain_id = dm.make_fk_id(source_domain)
 
             db.from(source_table)
               .where(source_domain_id => source_ids)
@@ -94,7 +101,7 @@ module ConceptQL
 
         unless from_me_domains.empty?
           from_me_domains.each do |from_me_domain|
-            fk_domain_id = make_domain_id(from_me_domain)
+            fk_domain_id = dm.fk_by_domain(from_me_domain)
             wheres << Sequel.expr(fk_domain_id => db.from(stream_query).where(criterion_domain: from_me_domain.to_s).select_group(:criterion_id))
           end
         end

data/lib/conceptql/operators/co_reported.rb

@@ -1,29 +1,99 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class CoReported < PassThru
       register __FILE__
 
-      desc 'Passes through all events that were co-reported in a database. For OMOP CDM, an event is considered to be "co-reported" if it shares the same visit_occurrence as another event.'
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Passes along all events that were co-reported in the same record in the source data."
+
       allows_many_upstreams
       category "Combine Streams"
       default_query_columns
       validate_at_least_one_upstream
       validate_no_arguments
-      require_column :visit_occurrence_id
+      require_column :context_id
 
       def query(db)
-        events_with_common_visits(db).inject { |q, query| q.union(query, all: true) }
+        if gdm?
+          gdm(db)
+        else
+          events_with_common_visits(db).inject { |q, query| q.union(query, all: true) }
+        end
+      end
+
+      def gdm(db)
+        contexteds = upstreams.map do |stream|
+          [ stream.cte_name(:contextified), contextify(db, stream) ]
+        end.to_h
+
+        # Get all context_id's that are in all streams and do not share the same critierion_id
+        first, *rest = *contexteds.keys
+        shared_context_ids = rest.inject(db[first].select(:context_id)) do |q, next_cte|
+          q.intersect(db[next_cte].select(:context_id))
+        end
+        # shared_context_ids = rest.inject(first.from_self(alias: :first)) do |q, shared_event|
+        #    q.join(shared_event.select(:context_id, :criterion_id), context_id: :context_id) do |a,b|
+        #    ~(Sequel.qualify(a,:criterion_id) =~ Sequel.qualify(b,:criterion_id) )
+        #  end.select(Sequel[:first][:context_id]).distinct
+        # end.from_self
+
+        if ConceptQL.avoid_ctes?
+          context_id_ds = shared_context_ids
+        else
+          name = cte_name(:shared_context_ids)
+          context_id_ds = db[name]
+        end
+
+        shared_events = contexteds.keys.map do |contexted|
+          db[contexted].where(context_id: context_id_ds).select(*query_cols)
+        end
+
+        ds = shared_events.inject do |q, shared_event|
+          q.union(shared_event)
+        end
+
+        contexteds.each do |name, query|
+          ds = ds.with(name, query)
+        end
+        if name
+          opts = {}
+          opts[:materialized] = true if rdbms.supports_materialized?
+          ds = ds.with(name, shared_context_ids, opts)
+        end
+
+        ds
+      end
+
+      def contextify(db, stream)
+        stream.evaluate(db).from_self(alias: :s)
+        # .join(Sequel[:clinical_codes].as(:c), id: :criterion_id)
+        # .select_all(:s)
+        # .select_append(Sequel[:c][:context_id].as(:context_id))
+        # .from_self
       end
 
       private
 
       def visit_occurrence_ids_in_common(db)
-        @visit_occurrence_ids_in_common ||= upstream_queries(db).map { |q| q.select(:visit_occurrence_id) }.inject do |q, query|
+        @visit_occurrence_ids_in_common ||= upstream_queries(db).map do |q|
+          q.select(:visit_occurrence_id)
+        end.inject do |q, query|
           q.from_self(alias: :tab1)
-            .join(query.as(:tab2), tab1__visit_occurrence_id: :tab2__visit_occurrence_id)
-            .select(:tab1__visit_occurrence_id___visit_occurrence_id)
+           .join(query.as(:tab2), visit_occurrence_id: :visit_occurrence_id)
+           .select(Sequel[:tab1][:visit_occurrence_id].as(:visit_occurrence_id))
         end
       end
 
@@ -33,10 +103,9 @@ module ConceptQL
 
       def upstream_queries(db)
         @upstream_queries ||= upstreams.map do |expression|
-          expression.evaluate(db).from_self
+          expression.evaluate(db).select(*query_cols).from_self
         end
       end
     end
   end
 end
-

data/lib/conceptql/operators/concurrent_within.rb

@@ -1,20 +1,32 @@
-require_relative 'operator'
-require_relative '../date_adjuster'
+# frozen_string_literal: true
+
+require_relative "operator"
+require_relative "../date_adjuster"
 
 module ConceptQL
   module Operators
     class ConcurrentWithin < Operator
       register __FILE__
 
-      desc 'Filters each upstream to only include rows where there are matching entries in each of the other upstreams.'
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Filters each upstream to only include records where other upstreams' date ranges overlap"
       option :start, type: :string
       option :end, type: :string
       allows_many_upstreams
       validate_at_least_one_upstream
       validate_no_arguments
       validate_option DateAdjuster::VALID_INPUT, :start, :end
-      category "Modify Data"
-      basic_type :temporal
+      category "Combine Streams"
+      basic_type :set
       default_query_columns
 
       def query(db)
@@ -24,20 +36,22 @@ module ConceptQL
 
         return datasets.first.from_self if datasets.length == 1
 
-        adjusted_start_date = DateAdjuster.new(options[:start]).adjust(:l__start_date, true)
-        adjusted_end_date = DateAdjuster.new(options[:end]).adjust(:l__end_date)
+        adjusted_start_date = DateAdjuster.new(self, options[:start]).adjust(Sequel[:l][:start_date], true)
+        adjusted_end_date = DateAdjuster.new(self, options[:end]).adjust(Sequel[:l][:end_date])
 
         datasets = datasets.map do |ds|
-          matching = ds.from_self(:alias=>:l)
+          matching = ds.from_self(alias: :l)
 
-          (datasets - [ds]).each do |other|
+          (datasets - [ ds ]).each do |other|
             other = other
-              .from_self(:alias=>:r)
-              .where(adjusted_start_date <= :r__start_date)
-              .where(adjusted_end_date >= :r__end_date)
-              .select(:person_id)
+                    .from_self(alias: :r)
+                    .where(adjusted_start_date <= Sequel[:r][:start_date])
+                    .where(adjusted_end_date >= Sequel[:r][:end_date])
+                    .select(*matching_columns)
 
-            matching = matching.where(:person_id=>other)
+            matching = matching.where(other.where(matching_columns.map do |x|
+              [ Sequel.qualify(:l, x), Sequel.qualify(:r, x) ]
+            end).exists)
           end
 
           matching
@@ -45,7 +59,7 @@ module ConceptQL
 
         ds, *rest = datasets
         rest.each do |other|
-          ds = ds.union(other, :from_self=>nil)
+          ds = ds.union(other, from_self: nil)
         end
 
         ds.from_self
@@ -53,5 +67,3 @@ module ConceptQL
     end
   end
 end
-
-

data/lib/conceptql/operators/contains.rb

@@ -1,20 +1,20 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class Contains < TemporalOperator
       register __FILE__
 
-      desc <<-EOF
-If a result in the left hand results (LHR) has a start_date on or before and an end_date on or after a result in the right hand results (RHR), it is passed through.
-L--X-L
-R-----R
-L------Y--------L
-
-      EOF
+      desc "For each person, passes along left hand records with a start_date and end_date containing a right hand record's start_date and end_date."
 
       def where_clause
-        [Proc.new { l__start_date <= r__start_date}, Proc.new { r__end_date <= l__end_date }]
+        (within_start <= r_start_date) & (r_end_date <= within_end)
+      end
+
+      def within_source_table
+        Sequel[:l]
       end
     end
   end

data/lib/conceptql/operators/count.rb

@@ -1,34 +1,34 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Count < PassThru
       register __FILE__
 
-      desc 'Counts the number of results the exactly match across all columns.'
+      desc "Counts the number of records that exactly match across all columns."
       allows_one_upstream
       validate_one_upstream
       validate_no_arguments
       require_column :value_as_number
 
       def query_cols
-        dynamic_columns - [:value_as_number] + [:value_as_number]
+        dynamic_columns - [ :value_as_number ] + [ :value_as_number ]
       end
 
       def query(db)
         db.from(unioned(db))
-          .select_group(*(query_cols - [:value_as_number]))
-          .select_append{count(1).as(:value_as_number)}
+          .select_group(*(query_cols - [ :value_as_number ]))
+          .select_append { count(1).as(:value_as_number) }
           .from_self
       end
 
       def unioned(db)
-        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
-          uni.union(q)
+        upstreams.map { |c| c.evaluate(db).select(*query_cols) }.inject do |uni, q|
+          uni.union(q.from_self)
         end
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/date_range.rb

@@ -1,4 +1,7 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
+require "date"
 
 module ConceptQL
   module Operators
@@ -10,34 +13,41 @@ module ConceptQL
     class DateRange < Operator
       register __FILE__
 
-      desc 'Used to represent a date literal.'
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      DATE_FORMAT = /\A#{Regexp.union([ /START/i, /END/i, /\d{4}-\d{2}-\d{2}/ ])}\z/.freeze
+
+      desc "Creates a set of all person records with the given start_date and end_date."
       option :start, type: :string
       option :end, type: :string
       category "Select by Property"
       basic_type :selection
       validate_no_upstreams
       validate_no_arguments
-      validate_option String, :start, :end
+      validate_option DATE_FORMAT, :start, :end
       validate_required_options :start, :end
 
-      def query_cols
-        table_columns(:person) + [:criterion_domain, :criterion_id, :start_date, :end_date]
-      end
-
       def query(db)
-        db.from(:person)
-          .select_append(Sequel.cast_string('person').as(:criterion_domain))
-          .select_append(Sequel.expr(:person_id).as(:criterion_id))
-          .select_append(Sequel.as(cast_date(db, start_date(db)), :start_date),
-                         Sequel.as(cast_date(db, end_date(db)), :end_date)).from_self
+        replace = {
+          start_date: start_date(db),
+          end_date: end_date(db)
+        }
+        db.from(source_table)
+          .select(*dm.columns(table: source_table, replace: replace))
+          .from_self
       end
 
-      def domains(db)
-        [:person]
+      def domains(_db)
+        [ :person ]
       end
 
       private
 
+      def source_table
+        dm.table_by_domain(:person)
+      end
+
       def start_date(db)
         date_from(db, options[:start])
       end
@@ -49,11 +59,23 @@ module ConceptQL
       # TODO: Select the earliest and latest dates of observation from
       # the proper CDM table to represent the start and end of data
       def date_from(db, str)
-        return db.from(:observation_period).get { min(:observation_period_start_date) } if str.upcase == 'START'
-        return db.from(:observation_period).get { max(:observation_period_end_date) } if str.upcase == 'END'
-        return str
+        return db.from(period_table).get { |o| o.min(period_start_date(db)) } if str.upcase == "START"
+        return db.from(period_table).get { |o| o.max(period_end_date(db)) } if str.upcase == "END"
+
+        str
+      end
+
+      def period_table
+        dm.period_table
+      end
+
+      def period_start_date(db)
+        dm.start_date_column(db, period_table)
+      end
+
+      def period_end_date(db)
+        dm.end_date_column(db, period_table)
       end
     end
   end
 end
-

data/lib/conceptql/operators/death.rb

@@ -1,11 +1,15 @@
-require_relative 'casting_operator'
+# frozen_string_literal: true
+
+require_relative "casting_operator"
 
 module ConceptQL
   module Operators
     class Death < CastingOperator
+      include ConceptQL::Behaviors::Windowable
+
       register __FILE__
 
-      desc 'Generates all death records, or, if fed a stream, fetches all death records for the people represented in the incoming result set.'
+      desc "Selects all death records"
       domains :death
       allows_one_upstream
 
@@ -13,6 +17,10 @@ module ConceptQL
         :death
       end
 
+      def source_table
+        dm.table_by_domain(:death)
+      end
+
       def i_point_at
         [ :person ]
       end

data/lib/conceptql/operators/dedup.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+
+module ConceptQL
+  module Operators
+    class Dedup < PassThru
+      register __FILE__
+
+      desc "Counts the number of records that exactly match across all columns."
+      allows_one_upstream
+      validate_one_upstream
+
+      def query(db)
+        db.from(unioned(db))
+          .select_group(*query_cols)
+          .from_self
+      end
+
+      def query_cols
+        dynamic_columns
+      end
+
+      def unioned(db)
+        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
+          uni.union(q)
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/operators/during.rb

@@ -1,23 +1,16 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class During < TemporalOperator
       register __FILE__
 
-      desc <<-EOF
-Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand results (RHR).
-Any result in the LHR with a start_date and end_date that occur within the start_date and end_date of a result in the RHR is passed through.
-All other results are discarded, including all results in the RHR.
-      EOF
+      desc "For each person, passes along left hand records with a start_date and end_date within a right hand record's start_date and end_date."
 
       def where_clause
-        if inclusive?
-          Sequel.expr(Sequel.expr(Proc.new { r__start_date <= l__start_date}).&(Sequel.expr( Proc.new { l__start_date <= r__end_date })))
-            .|(Sequel.expr(Proc.new { r__start_date <= l__end_date}).&(Sequel.expr( Proc.new { l__end_date <= r__end_date })))
-        else
-          [Proc.new { r__start_date <= l__start_date}, Proc.new { l__end_date <= r__end_date }]
-        end
+        (within_start <= l_start_date) & (l_end_date <= within_end)
       end
     end
   end