conceptql 0.2.0 → 0.4.0

data/lib/conceptql/operators/temporal_operator.rb

@@ -1,36 +1,128 @@
-require_relative 'binary_operator_operator'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
+require_relative "../date_adjuster"
 
 module ConceptQL
   module Operators
     # Base class for all temporal operators
     #
     # Subclasses must implement the where_clause method which should probably return
-    # a proc that can be executed as a Sequel "virtual row" e.g.
-    # Proc.new { l.end_date < r.start_date }
+    # a Sequel expression to use for filtering.
     class TemporalOperator < BinaryOperatorOperator
       reset_categories
-      category %w(Temporal Relative)
+      category "Filter by Comparing"
+      default_query_columns
+
+      option :within, type: :string,
+                      instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+
+      # The temporal operators read join_columns through
+      # BinaryOperatorOperator#join_columns_option, but do not declare it with
+      # `option`: only t_shank gives it, and a declared option would appear in
+      # ConceptQL.metadata, so the diagram editor would offer it (Ryan,
+      # 2026-10-26).
+
+      validate_option DateAdjuster::VALID_INPUT, :within, :at_least
+
+      class << self
+        def allows_at_least_option
+          option :at_least, type: :string,
+                            instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+        end
+      end
+
+      def self.within_skip(type)
+        define_method(:"within_check_#{type}?") { false }
+      end
 
       def query(db)
-        db.from(db.from(left_stream(db))
-                  .join(right_stream(db), l__person_id: :r__person_id)
-                  .where(where_clause)
-                  .select_all(:l))
+        right_stream = apply_selectors(right_stream_query(db), function: rhs_function).from_self(alias: :r)
+        ds = semi_or_inner_join(left_stream(db), right_stream, *join_columns, where_clause)
+
+        ds = apply_selectors(ds, qualifier: :r)
+
+        ds.from_self
+      end
+
+      def r_start_date
+        Sequel[:r][:start_date]
+      end
+
+      def r_end_date
+        Sequel[:r][:end_date]
+      end
+
+      def l_start_date
+        Sequel[:l][:start_date]
+      end
+
+      def l_end_date
+        Sequel[:l][:end_date]
+      end
+
+      def within_option
+        return unless (v = options[:within])
+        return if v.strip.empty?
+
+        v
+      end
+
+      def at_least_option
+        return unless (v = options[:at_least])
+        return if v.strip.empty?
+
+        v
+      end
+
+      def within_source_table
+        Sequel[:r]
+      end
+
+      def within_start
+        within_start = within_source_table[:start_date]
+        within_start = adjust_date(within_option, within_start, true) if within_option
+        within_start
+      end
+
+      def within_end
+        within_end = within_source_table[:end_date]
+        within_end = adjust_date(within_option, within_end) if within_option
+        within_end
       end
 
-      def inclusive?
-        options[:inclusive]
+      def adjust_date(adjustment, column, reverse = false)
+        adjuster = DateAdjuster.new(self, adjustment)
+        adjuster.adjust(column, reverse)
       end
 
-      def left_stream(db)
-        Sequel.expr(left.evaluate(db)).as(:l)
+      def apply_selectors(ds, opts = {})
+        return ds unless include_rhs_columns
+
+        ds = ds.select_remove(*include_rhs_columns).select_append(*rhs_columns(opts)) if include_rhs_columns
+        ds
+      end
+
+      def rhs_columns(opts)
+        cols = include_rhs_columns
+        return cols if opts.empty?
+
+        cols = cols.map { |c| Sequel.function(opts[:function], c).as(c) } if opts[:function]
+        cols = cols.map { |c| Sequel[opts[:qualifier]][c].as(c) } if opts[:qualifier]
+        cols
+      end
+
+      def include_rhs_columns
+        options[:include_rhs_columns]&.map(&:to_sym)
+      end
+
+      def rhs_function
+        nil
       end
 
-      def right_stream(db)
-        Sequel.expr(right.evaluate(db)).as(:r)
+      def use_inner_join?
+        super || options[:include_rhs_columns]
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/time_window.rb

@@ -1,5 +1,7 @@
-require_relative 'operator'
-require_relative '../date_adjuster'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+require_relative "../date_adjuster"
 
 module ConceptQL
   module Operators
@@ -19,22 +21,27 @@ module ConceptQL
     # Both start and end arguments must be provided, but if you do not wish to adjust a date just
     # pass '', '0', or nil as that argument.  E.g.:
     # start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
-    class TimeWindow < Operator
-      desc 'Adjusts the start_date and end_date columns to create a new window of time for each result.'
-      option :start, type: :string
-      option :end, type: :string
+    class TimeWindow < PassThru
+      register __FILE__
+
+      desc "Adjusts start_date and end_date to create a new window of time for each record."
+      option :start, type: :string,
+                     instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date.'
+      option :end, type: :string,
+                   instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date.'
       allows_one_upstream
-      category %(Temporal Manipulation)
+      validate_one_upstream
+      validate_no_arguments
+      validate_option DateAdjuster::VALID_INPUT, :start, :end
+      category "Modify Data"
+      basic_type :temporal
 
       def query(db)
-        db.extension :date_arithmetic
-        db.from(stream.evaluate(db))
+        dm.selectify(db.from(stream.evaluate(db)).from_self,
+                     replace: { start_date: adjusted_start_date, end_date: adjusted_end_date })
       end
 
       private
-      def date_columns(query, type = nil)
-        [adjusted_start_date, adjusted_end_date]
-      end
 
       def adjusted_start_date
         adjusted_date(:start, :start_date)
@@ -44,28 +51,9 @@ module ConceptQL
         adjusted_date(:end, :end_date)
       end
 
-      def manipulator
-        @manipulator ||= options.fetch(:date_manipulator, Sequel)
-      end
-
-      # NOTE: This produces PostgreSQL-specific date adjustment.  I'm not yet certain how to generalize this
-      # or make different versions based on RDBMS
       def adjusted_date(option_arg, column)
-        arg = options[option_arg]
-        arg ||= ''
-        return ['end_date', column].join('___').to_sym if arg.downcase == 'end'
-        return ['start_date', column].join('___').to_sym if arg.downcase == 'start'
-        return Sequel.cast(Date.parse(arg).strftime('%Y-%m-%d'), Date).as(column) if arg =~ /^\d{4}-\d{2}-\d{2}$/
-        adjusted_date = DateAdjuster.new(arg).adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
-          if quantity > 0
-            manipulator.date_add(sql, units => quantity)
-          else
-            manipulator.date_sub(sql, units => quantity.abs)
-          end
-        end
-        Sequel.cast(adjusted_date, Date).as(column)
+        DateAdjuster.new(self, options[option_arg], manipulator: options[:manipulator]).adjust(column)
       end
     end
   end
 end
-

data/lib/conceptql/operators/trim_date.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
+
+module ConceptQL
+  module Operators
+    # Trims the end_date of the LHS set of records by the RHS's earliest
+    # start_date (per person)
+    # If a the RHS contains a start_date that comes before the LHS's start_date
+    # that LHS record is completely discarded.
+    #
+    # If there is no RHS record for an LHS record, the LHS record is passed
+    # thru unaffected.
+    #
+    # If the RHS record's start_date is later than the LHS end_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDate < TemporalOperator
+      desc <<~EOF
+        Trims the end_date of the left hand records (LHR) by the earliest
+        start_date (per person) in the right hand records (RHR)
+        If the RHR contains a start_date that comes before the start_date in the LHR
+        that record in the LHR is completely discarded.
+
+        If there is no record in the RHR for a record in the LHR, the record in the LHR is passed
+        through unaffected.
+
+        If the start_date of the record in the RHR is later than the end_date of the record in the LHR, the record in the LHR
+        is passed through unaffected.
+      EOF
+
+      allows_one_upstream
+
+      def query(db)
+        # If the RHS's min start date is less than the LHS start date,
+        # the entire LHS date range is truncated, which implies the row itself
+        # is ineligible to pass thru
+        ds = db.from(left_stream(db)).from_self(alias: :l)
+               .left_join(Sequel.as(right_stream_query(db), :r), join_columns.inject(&:&))
+               .where(where_criteria)
+
+        ds = dm.selectify(ds, qualifier: :l, replace: replacement_columns)
+
+        ds = apply_selectors(ds)
+
+        ds.from_self
+      end
+
+      private
+
+      def compare_all?
+        options[:compare_all]
+      end
+
+      def right_stream_query(db)
+        rh = if compare_all?
+               super
+        else
+               occ = to_op([ :occurrence, occurrence_number, right ])
+               occ.evaluate(db)
+        end
+        columnizer.apply(rh)
+      end
+
+      def columnizer
+        columnizer = Columnizer.new
+        columnizer.add_columns(:person_id, trim_date, *join_columns_option)
+        columnizer
+      end
+    end
+  end
+end

data/lib/conceptql/operators/trim_date_end.rb

@@ -1,55 +1,43 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "trim_date"
 
 module ConceptQL
   module Operators
-    # Trims the end_date of the LHS set of results by the RHS's earliest
+    # Trims the end_date of the LHS set of records by the RHS's earliest
     # start_date (per person)
     # If a the RHS contains a start_date that comes before the LHS's start_date
-    # that LHS result is completely discarded.
+    # that LHS record is completely discarded.
     #
-    # If there is no RHS result for an LHS result, the LHS result is passed
+    # If there is no RHS record for an LHS record, the LHS record is passed
     # thru unaffected.
     #
-    # If the RHS result's start_date is later than the LHS end_date, the LHS
-    # result is passed thru unaffected.
-    class TrimDateEnd < TemporalOperator
-      desc <<-EOF
-Trims the end_date of the LHS set of results by the RHS's earliest
-start_date (per person)
-If a the RHS contains a start_date that comes before the LHS's start_date
-that LHS result is completely discarded.
-
-If there is no RHS result for an LHS result, the LHS result is passed
-thru unaffected.
-
-If the RHS result's start_date is later than the LHS end_date, the LHS
-result is passed thru unaffected.
-      EOF
+    # If the RHS record's start_date is later than the LHS end_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDateEnd < TrimDate
+      register __FILE__
+
+      desc "For each person, trims the end_date of all left hand records by the earliest start_date in the right hand records."
+
       allows_one_upstream
-      category %(Temporal Manipulation)
-
-      def query(db)
-        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:min, :start_date), :start_date))
-
-        where_criteria = Sequel.expr { l__start_date <= r__start_date }
-        where_criteria = where_criteria.|(r__start_date: nil)
-
-        # If the RHS's min start date is less than the LHS start date,
-        # the entire LHS date range is truncated, which implies the row itself
-        # is ineligible to pass thru
-        db.from(db.from(left_stream(db))
-                  .left_join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
-                  .where(where_criteria)
-                  .select(*new_columns)
-                  .select_append(Sequel.as(Sequel.function(:least, :l__end_date, :r__start_date), :end_date))
-               )
-      end
 
       private
-      def new_columns
-        (COLUMNS - [:end_date]).map { |col| "l__#{col}".to_sym }
+
+      def trim_date
+        :start_date
+      end
+
+      def where_criteria
+        (l_start_date <= within_start) | { r_start_date => nil }
+      end
+
+      def replacement_columns
+        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, within_start, l_end_date)) }
+      end
+
+      def occurrence_number
+        1
       end
     end
   end
 end
-

data/lib/conceptql/operators/trim_date_start.rb

@@ -1,56 +1,43 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "trim_date"
 
 module ConceptQL
   module Operators
-    # Trims the start_date of the LHS set of results by the RHS's latest
+    # Trims the start_date of the LHS set of records by the RHS's latest
     # end_date (per person)
     # If a the RHS contains an end_date that comes after the LHS's end_date
-    # that LHS result is completely discarded.
+    # that LHS record is completely discarded.
     #
-    # If there is no RHS result for an LHS result, the LHS result is passed
+    # If there is no RHS record for an LHS record, the LHS record is passed
     # thru unaffected.
     #
-    # If the RHS result's end_date is earlier than the LHS start_date, the LHS
-    # result is passed thru unaffected.
-    class TrimDateStart < TemporalOperator
-      desc <<-EOF
-Trims the start_date of the LHS set of results by the RHS's latest
-end_date (per person)
-If a the RHS contains an end_date that comes after the LHS's end_date
-that LHS result is completely discarded.
-
-If there is no RHS result for an LHS result, the LHS result is passed
-thru unaffected.
-
-If the RHS result's end_date is earlier than the LHS start_date, the LHS
-result is passed thru unaffected.
-      EOF
+    # If the RHS record's end_date is earlier than the LHS start_date, the LHS
+    # record is passed thru unaffected.
+    class TrimDateStart < TrimDate
+      register __FILE__
+
+      desc "For each person, trims the start_date of all left hand records by the most recent end_date in the right hand records."
+
       allows_one_upstream
-      category %(Temporal Manipulation)
-
-      def query(db)
-        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:max, :end_date), :end_date))
-
-        where_criteria = Sequel.expr { l__end_date >= r__end_date }
-        where_criteria = where_criteria.|(r__end_date: nil)
-
-        # If the RHS's min start date is less than the LHS start date,
-        # the entire LHS date range is truncated, which implies the row itself
-        # is ineligible to pass thru
-        db.from(db.from(left_stream(db))
-                  .join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
-                  .where(where_criteria)
-                  .select(*new_columns)
-                  .select_append(Sequel.as(Sequel.function(:greatest, :l__start_date, :r__end_date), :start_date))
-               )
-      end
 
       private
-      def new_columns
-        (COLUMNS - [:start_date]).map { |col| "l__#{col}".to_sym }
+
+      def trim_date
+        :end_date
+      end
+
+      def where_criteria
+        (l_end_date >= within_end) | { r_end_date => nil }
+      end
+
+      def replacement_columns
+        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, within_end, l_start_date)) }
+      end
+
+      def occurrence_number
+        -1
       end
     end
   end
 end
-
-

data/lib/conceptql/operators/union.rb

@@ -1,16 +1,77 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     class Union < PassThru
-      desc 'Pools sets of incoming results into a single large set of results.'
+      register __FILE__
+
+      class Combiner
+        attr_reader :combinables
+
+        def initialize(combinables)
+          @combinables = combinables
+        end
+
+        def queries(db)
+          combinables.group_by(&:domain).map do |domain, combos|
+            include_uuid = combos.any? { |c| c.options[:uuid] }
+            criteria = combos.map do |combo|
+              combo.filter_clause(db)
+            end.inject(&:|)
+
+            criteria = Sequel.&(vocabularies_in(combos), criteria) if combos.length > 1
+
+            table = combos.first.criterion_table
+            ds = db[table]
+                 .where(criteria)
+            combos.first.select_it(ds, specific_table: table, domain: domain, uuid: include_uuid)
+          end
+        end
+
+        private
+
+        # Every OR'd criterion already requires its own vocabulary, so this IN
+        # list selects no new rows and drops none. It is explicit because a
+        # plain OR stopped DuckDB's Parquet filter pushdown (bead conceptql-agx).
+        def vocabularies_in(combos)
+          conditions = combos.map(&:vocabulary_condition)
+          columns = conditions.flat_map(&:keys).uniq
+          raise "Merged vocabularies use more than one vocabulary column: #{columns}" unless columns.length == 1
+
+          { columns.first => conditions.map { |c| c[columns.first] }.uniq }
+        end
+      end
+
+      def events_per_patient
+        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
+      end
+
+      def multiple_vocabularies
+        return true if upstreams.any?(&:multiple_vocabularies)
+
+        vocabularies.length > 1
+      end
+
+      desc "Combines sets of incoming records into a single large set of records."
       allows_many_upstreams
-      category 'Set Logic'
+      category "Combine Streams"
+      default_query_columns
+      validate_at_least_one_upstream
+      validate_no_arguments
 
       def query(db)
-        values.map do |expression|
-          expression.evaluate(db).from_self
-        end.inject do |q, query|
+        combinables, individuals = upstreams.partition { |upstream| upstream.is_a?(Vocabulary) }
+
+        queries = individuals.map do |expression|
+          expression.evaluate(db)
+        end
+
+        queries += Combiner.new(combinables).queries(db)
+
+        queries.map! { |ds| ds.from_self.select(*query_cols).from_self }
+        queries.inject do |q, query|
           q.union(query, all: true)
         end
       end