conceptql 0.3.0 → 0.4.0

data/lib/conceptql/operators/snf.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Snf < Operator
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
+      include ConceptQL::Behaviors::Utilizable
+
+      desc "Selects admission records of type 'SNF'"
+
+      def collection_type
+        "SNF"
+      end
+    end
+  end
+end

data/lib/conceptql/operators/sum.rb

@@ -1,13 +1,25 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Sum < PassThru
       register __FILE__
 
-      desc <<-EOF
-Sums value_as_number across all results that match on all but start_date, end_date.
-For start_date and end_date the min and max of each respectively is returned.'
+      def events_per_patient
+        :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc <<~EOF
+        Sums value_as_number across all records that match on all but start_date, end_date.
+        For start_date and end_date the min and max of each respectively is returned.'
       EOF
       default_query_columns
       validate_at_least_one_upstream
@@ -16,20 +28,19 @@ For start_date and end_date the min and max of each respectively is returned.'
 
       def query(db)
         db.from(unioned(db))
-          .select_group(*(dynamic_columns - [:start_date, :end_date, :criterion_id, :value_as_number]))
-          .select_append(Sequel.lit('?', 0).as(:criterion_id))
-          .select_append{ min(start_date).as(:start_date) }
-          .select_append{ max(end_date).as(:end_date) }
-          .select_append{sum(value_as_number).as(:value_as_number)}
+          .select_group(*(dynamic_columns - %i[start_date end_date criterion_id value_as_number]))
+          .select_append(Sequel.lit("?", 0).as(:criterion_id))
+          .select_append { min(start_date).as(:start_date) }
+          .select_append { max(end_date).as(:end_date) }
+          .select_append { sum(value_as_number).as(:value_as_number) }
           .from_self
       end
 
       def unioned(db)
-        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
-          uni.union(q)
+        upstreams.map { |c| c.evaluate(db).select(*query_cols) }.inject do |uni, q|
+          uni.union(q.from_self)
         end
       end
     end
   end
 end
-

data/lib/conceptql/operators/temporal_operator.rb

@@ -1,5 +1,7 @@
-require_relative 'binary_operator_operator'
-require_relative '../date_adjuster'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
+require_relative "../date_adjuster"
 
 module ConceptQL
   module Operators
@@ -12,91 +14,115 @@ module ConceptQL
       category "Filter by Comparing"
       default_query_columns
 
-      option :within, type: :string, instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
-      option :at_least, type: :string, instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
-      option :occurrences, type: :integer, desc: "Number of occurrences that must precede the event of interest, e.g. if you'd like the 4th event in a set of events, set occurrences to 3"
+      option :within, type: :string,
+                      instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+
+      # The temporal operators read join_columns through
+      # BinaryOperatorOperator#join_columns_option, but do not declare it with
+      # `option`: only t_shank gives it, and a declared option would appear in
+      # ConceptQL.metadata, so the diagram editor would offer it (Ryan,
+      # 2026-10-26).
 
       validate_option DateAdjuster::VALID_INPUT, :within, :at_least
-      validate_option /\A\d+\z/, :occurrences
+
+      class << self
+        def allows_at_least_option
+          option :at_least, type: :string,
+                            instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+        end
+      end
 
       def self.within_skip(type)
-        define_method(:"within_check_#{type}?"){false}
+        define_method(:"within_check_#{type}?") { false }
       end
 
       def query(db)
-        ds = db.from(left_stream(db))
-               .join(right_stream(db), l__person_id: :r__person_id)
-               .where(where_clause)
-               .select_all(:l)
+        right_stream = apply_selectors(right_stream_query(db), function: rhs_function).from_self(alias: :r)
+        ds = semi_or_inner_join(left_stream(db), right_stream, *join_columns, where_clause)
+
+        ds = apply_selectors(ds, qualifier: :r)
 
-        ds = add_option_conditions(ds)
         ds.from_self
       end
 
-      def add_option_conditions(ds)
-        if within = options[:within]
-          ds = add_within_condition(ds, within)
-        end
+      def r_start_date
+        Sequel[:r][:start_date]
+      end
 
-        if at_least = options[:at_least]
-          ds = add_within_condition(ds, at_least, :exclude)
-        end
+      def r_end_date
+        Sequel[:r][:end_date]
+      end
 
-        if occurrences = options[:occurrences]
-          ds = add_occurrences_condition(ds, occurrences)
-        end
+      def l_start_date
+        Sequel[:l][:start_date]
+      end
 
-        ds
+      def l_end_date
+        Sequel[:l][:end_date]
+      end
+
+      def within_option
+        return unless (v = options[:within])
+        return if v.strip.empty?
+
+        v
+      end
+
+      def at_least_option
+        return unless (v = options[:at_least])
+        return if v.strip.empty?
+
+        v
       end
 
-      def add_within_condition(ds, within, meth=:where)
-        within = DateAdjuster.new(within)
-        after = within.adjust(:r__start_date, true)
-        before = within.adjust(:r__end_date)
-        within_col = Sequel.expr(within_column)
-        ds = ds.send(meth){within_col >= after} if within_check_after?
-        ds = ds.send(meth){within_col <= before} if within_check_before?
-        ds.distinct
+      def within_source_table
+        Sequel[:r]
       end
 
-      def add_occurrences_condition(ds, occurrences)
-        occurrences_col = occurrences_column
-        ds.distinct.from_self
-          .select_append{row_number{}.over(:partition => :person_id, :order => occurrences_col).as(:occurrence)}
-          .from_self
-          .select(*query_columns(ds))
-          .where{occurrence > occurrences.to_i}
+      def within_start
+        within_start = within_source_table[:start_date]
+        within_start = adjust_date(within_option, within_start, true) if within_option
+        within_start
       end
 
-      def within_column
-        :l__start_date
+      def within_end
+        within_end = within_source_table[:end_date]
+        within_end = adjust_date(within_option, within_end) if within_option
+        within_end
       end
 
-      def occurrences_column
-        :start_date
+      def adjust_date(adjustment, column, reverse = false)
+        adjuster = DateAdjuster.new(self, adjustment)
+        adjuster.adjust(column, reverse)
       end
 
-      def within_check_after?
-        true
+      def apply_selectors(ds, opts = {})
+        return ds unless include_rhs_columns
+
+        ds = ds.select_remove(*include_rhs_columns).select_append(*rhs_columns(opts)) if include_rhs_columns
+        ds
       end
 
-      def within_check_before?
-        true
+      def rhs_columns(opts)
+        cols = include_rhs_columns
+        return cols if opts.empty?
+
+        cols = cols.map { |c| Sequel.function(opts[:function], c).as(c) } if opts[:function]
+        cols = cols.map { |c| Sequel[opts[:qualifier]][c].as(c) } if opts[:qualifier]
+        cols
       end
 
-      def inclusive?
-        options[:inclusive]
+      def include_rhs_columns
+        options[:include_rhs_columns]&.map(&:to_sym)
       end
 
-      def left_stream(db)
-        Sequel.expr(left.evaluate(db).from_self).as(:l)
+      def rhs_function
+        nil
       end
 
-      def right_stream(db)
-        Sequel.expr(right.evaluate(db).from_self).as(:r)
+      def use_inner_join?
+        super || options[:include_rhs_columns]
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/time_window.rb

@@ -1,5 +1,7 @@
-require_relative 'operator'
-require_relative '../date_adjuster'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+require_relative "../date_adjuster"
 
 module ConceptQL
   module Operators
@@ -19,30 +21,28 @@ module ConceptQL
     # Both start and end arguments must be provided, but if you do not wish to adjust a date just
     # pass '', '0', or nil as that argument.  E.g.:
     # start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
-    class TimeWindow < Operator
+    class TimeWindow < PassThru
       register __FILE__
 
-      desc 'Adjusts the start_date and end_date to create a new window of time for each result.'
-      option :start, type: :string
-      option :end, type: :string
+      desc "Adjusts start_date and end_date to create a new window of time for each record."
+      option :start, type: :string,
+                     instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date.'
+      option :end, type: :string,
+                   instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date.'
       allows_one_upstream
       validate_one_upstream
       validate_no_arguments
       validate_option DateAdjuster::VALID_INPUT, :start, :end
       category "Modify Data"
       basic_type :temporal
-      default_query_columns
 
       def query(db)
-        db.from(stream.evaluate(db))
+        dm.selectify(db.from(stream.evaluate(db)).from_self,
+                     replace: { start_date: adjusted_start_date, end_date: adjusted_end_date })
       end
 
       private
 
-      def date_columns(query, domain = nil)
-        [adjusted_start_date, adjusted_end_date]
-      end
-
       def adjusted_start_date
         adjusted_date(:start, :start_date)
       end
@@ -52,10 +52,8 @@ module ConceptQL
       end
 
       def adjusted_date(option_arg, column)
-        adjusted_date = DateAdjuster.new(options[option_arg], manipulator: options[:manipulator]).adjust(column)
-        adjusted_date.as(column)
+        DateAdjuster.new(self, options[option_arg], manipulator: options[:manipulator]).adjust(column)
       end
     end
   end
 end
-

data/lib/conceptql/operators/trim_date.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
+
+module ConceptQL
+  module Operators
+    # Trims the end_date of the LHS set of records by the RHS's earliest
+    # start_date (per person)
+    # If a the RHS contains a start_date that comes before the LHS's start_date
+    # that LHS record is completely discarded.
+    #
+    # If there is no RHS record for an LHS record, the LHS record is passed
+    # thru unaffected.
+    #
+    # If the RHS record's start_date is later than the LHS end_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDate < TemporalOperator
+      desc <<~EOF
+        Trims the end_date of the left hand records (LHR) by the earliest
+        start_date (per person) in the right hand records (RHR)
+        If the RHR contains a start_date that comes before the start_date in the LHR
+        that record in the LHR is completely discarded.
+
+        If there is no record in the RHR for a record in the LHR, the record in the LHR is passed
+        through unaffected.
+
+        If the start_date of the record in the RHR is later than the end_date of the record in the LHR, the record in the LHR
+        is passed through unaffected.
+      EOF
+
+      allows_one_upstream
+
+      def query(db)
+        # If the RHS's min start date is less than the LHS start date,
+        # the entire LHS date range is truncated, which implies the row itself
+        # is ineligible to pass thru
+        ds = db.from(left_stream(db)).from_self(alias: :l)
+               .left_join(Sequel.as(right_stream_query(db), :r), join_columns.inject(&:&))
+               .where(where_criteria)
+
+        ds = dm.selectify(ds, qualifier: :l, replace: replacement_columns)
+
+        ds = apply_selectors(ds)
+
+        ds.from_self
+      end
+
+      private
+
+      def compare_all?
+        options[:compare_all]
+      end
+
+      def right_stream_query(db)
+        rh = if compare_all?
+               super
+        else
+               occ = to_op([ :occurrence, occurrence_number, right ])
+               occ.evaluate(db)
+        end
+        columnizer.apply(rh)
+      end
+
+      def columnizer
+        columnizer = Columnizer.new
+        columnizer.add_columns(:person_id, trim_date, *join_columns_option)
+        columnizer
+      end
+    end
+  end
+end

data/lib/conceptql/operators/trim_date_end.rb

@@ -1,63 +1,43 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "trim_date"
 
 module ConceptQL
   module Operators
-    # Trims the end_date of the LHS set of results by the RHS's earliest
+    # Trims the end_date of the LHS set of records by the RHS's earliest
     # start_date (per person)
     # If a the RHS contains a start_date that comes before the LHS's start_date
-    # that LHS result is completely discarded.
+    # that LHS record is completely discarded.
     #
-    # If there is no RHS result for an LHS result, the LHS result is passed
+    # If there is no RHS record for an LHS record, the LHS record is passed
     # thru unaffected.
     #
-    # If the RHS result's start_date is later than the LHS end_date, the LHS
-    # result is passed thru unaffected.
-    class TrimDateEnd < TemporalOperator
+    # If the RHS record's start_date is later than the LHS end_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDateEnd < TrimDate
       register __FILE__
 
-      desc <<-EOF
-Trims the end_date of the left hand results (LHR) by the earliest
-start_date (per person) in the right hand results (RHR)
-If the RHR contain a start_date that comes before the start_date in the LHR
-that result in the LHR is completely discarded.
-
-If there is no result in the RHR for a result in the LHR, the result in the LHR is passed
-through unaffected.
+      desc "For each person, trims the end_date of all left hand records by the earliest start_date in the right hand records."
 
-If the start_date of the result in the RHR is later than the end_date of the result in the LHR, the result in the LHR
-is passed through unaffected.
-      EOF
       allows_one_upstream
-      within_skip :before
 
-      def query(db)
-        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:min, :start_date), :start_date))
-
-        where_criteria = Sequel.expr { l__start_date <= r__start_date }
-        where_criteria = where_criteria.|(r__start_date: nil)
+      private
 
-        # If the RHS's min start date is less than the LHS start date,
-        # the entire LHS date range is truncated, which implies the row itself
-        # is ineligible to pass thru
-        ds = db.from(left_stream(db))
-                  .left_join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
-                  .where(where_criteria)
-                  .select(*new_columns)
-                  .select_append(Sequel.as(Sequel.function(:least, :l__end_date, :r__start_date), :end_date))
+      def trim_date
+        :start_date
+      end
 
-        ds = add_option_conditions(ds)
-        ds.from_self
+      def where_criteria
+        (l_start_date <= within_start) | { r_start_date => nil }
       end
 
-      def within_column
-        :l__end_date
+      def replacement_columns
+        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, within_start, l_end_date)) }
       end
 
-      private
-      def new_columns
-        (dynamic_columns - [:end_date]).map { |col| "l__#{col}".to_sym }
+      def occurrence_number
+        1
       end
     end
   end
 end
-

data/lib/conceptql/operators/trim_date_start.rb

@@ -1,66 +1,43 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "trim_date"
 
 module ConceptQL
   module Operators
-    # Trims the start_date of the LHS set of results by the RHS's latest
+    # Trims the start_date of the LHS set of records by the RHS's latest
     # end_date (per person)
     # If a the RHS contains an end_date that comes after the LHS's end_date
-    # that LHS result is completely discarded.
+    # that LHS record is completely discarded.
     #
-    # If there is no RHS result for an LHS result, the LHS result is passed
+    # If there is no RHS record for an LHS record, the LHS record is passed
     # thru unaffected.
     #
-    # If the RHS result's end_date is earlier than the LHS start_date, the LHS
-    # result is passed thru unaffected.
-    class TrimDateStart < TemporalOperator
+    # If the RHS record's end_date is earlier than the LHS start_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDateStart < TrimDate
       register __FILE__
 
-      desc <<-EOF
-Trims the start_date of the left hand results (LHR) by the final
-end_date (per person) in the right hand results (RHR)
-If the RHR contain an end_date that comes after the end_date in the LHR
-that result in the LHR is completely discarded.
-
-If there is no result in the RHR for a result in the LHR, the result in the LHR is passed
-through unaffected.
-
-If the end_date of the result in the RHR is earlier than the start_date of the result in the LHR, the result in the LHR
-is passed through unaffected.
-      EOF
+      desc "For each person, trims the start_date of all left hand records by the most recent end_date in the right hand records."
 
       allows_one_upstream
-      within_skip :after
 
-      def query(db)
-        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:max, :end_date), :end_date))
-
-        where_criteria = Sequel.expr { l__end_date >= r__end_date }
-        where_criteria = where_criteria.|(r__end_date: nil)
-
-        # If the RHS's min start date is less than the LHS start date,
-        # the entire LHS date range is truncated, which implies the row itself
-        # is ineligible to pass thru
-        ds = db.from(left_stream(db))
-                  .join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
-                  .where(where_criteria)
-                  .select(*new_columns)
-                  .select_append(Sequel.as(Sequel.function(:greatest, :l__start_date, :r__end_date), :start_date))
+      private
 
-        ds = add_option_conditions(ds)
-        ds.from_self
+      def trim_date
+        :end_date
       end
 
-      private
+      def where_criteria
+        (l_end_date >= within_end) | { r_end_date => nil }
+      end
 
-      def occurrences_column
-        :end_date
+      def replacement_columns
+        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, within_end, l_start_date)) }
       end
 
-      def new_columns
-        (dynamic_columns - [:start_date]).map { |col| "l__#{col}".to_sym }
+      def occurrence_number
+        -1
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/union.rb

@@ -1,11 +1,60 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Union < PassThru
       register __FILE__
 
-      desc 'Pools sets of incoming results into a single large set of results.'
+      class Combiner
+        attr_reader :combinables
+
+        def initialize(combinables)
+          @combinables = combinables
+        end
+
+        def queries(db)
+          combinables.group_by(&:domain).map do |domain, combos|
+            include_uuid = combos.any? { |c| c.options[:uuid] }
+            criteria = combos.map do |combo|
+              combo.filter_clause(db)
+            end.inject(&:|)
+
+            criteria = Sequel.&(vocabularies_in(combos), criteria) if combos.length > 1
+
+            table = combos.first.criterion_table
+            ds = db[table]
+                 .where(criteria)
+            combos.first.select_it(ds, specific_table: table, domain: domain, uuid: include_uuid)
+          end
+        end
+
+        private
+
+        # Every OR'd criterion already requires its own vocabulary, so this IN
+        # list selects no new rows and drops none. It is explicit because a
+        # plain OR stopped DuckDB's Parquet filter pushdown (bead conceptql-agx).
+        def vocabularies_in(combos)
+          conditions = combos.map(&:vocabulary_condition)
+          columns = conditions.flat_map(&:keys).uniq
+          raise "Merged vocabularies use more than one vocabulary column: #{columns}" unless columns.length == 1
+
+          { columns.first => conditions.map { |c| c[columns.first] }.uniq }
+        end
+      end
+
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Combines sets of incoming records into a single large set of records."
       allows_many_upstreams
       category "Combine Streams"
       default_query_columns
@@ -13,43 +62,18 @@ module ConceptQL
       validate_no_arguments
 
       def query(db)
-        upstreams.map do |expression|
-          expression.evaluate(db).from_self
-        end.inject do |q, query|
-          q.union(query, all: true)
-        end
-      end
+        combinables, individuals = upstreams.partition { |upstream| upstream.is_a?(Vocabulary) }
 
-      def flattened
-        exprs = []
-        upstreams.each do |x|
-          if x.is_a?(Union)
-            exprs.concat x.flattened.upstreams
-          else
-            exprs << x
-          end
+        queries = individuals.map do |expression|
+          expression.evaluate(db)
         end
-        dup_values(exprs)
-      end
 
-      def optimized
-        first, *rest = flattened.upstreams
-        exprs = [first]
-
-        rest.each do |expression|
-          add = true
-          exprs.length.times do |i|
-            exp = exprs[i]
-            if exprs[i].unionable?(expression)
-              exprs[i] = exp.union(expression, all: true)
-              add = false
-              break
-            end
-          end
-          exprs << expression if add
-        end
+        queries += Combiner.new(combinables).queries(db)
 
-        dup_values(exprs.map{|x| x.is_a?(Operator) ? x.optimized : x})
+        queries.map! { |ds| ds.from_self.select(*query_cols).from_self }
+        queries.inject do |q, query|
+          q.union(query, all: true)
+        end
       end
     end
   end