conceptql 0.2.0 → 0.4.0

data/lib/conceptql/operators/match.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
+
+module ConceptQL
+  module Operators
+    # Compares the columns from #columns. Without options, that is every query
+    # column, so a stream matched with itself keeps every record.
+    class Match < BinaryOperatorOperator
+      register __FILE__
+
+      desc "Keeps left hand records that have the same values as a right hand side record. By default, every column is compared. only_columns or except_columns narrows that set, and join_columns compares only the person and the listed columns. Two NULL values count as the same value."
+      default_query_columns
+
+      # match reads only_columns, except_columns, join_columns and
+      # invert_match, but does not declare them with `option`: only t_shank
+      # gives them, and a declared option would appear in ConceptQL.metadata,
+      # so the diagram editor would offer it (Ryan, 2026-10-26). Each column
+      # list is an array of column names or a comma-separated string (see
+      # Operator#column_names_option).
+
+      # These columns identify a record and are never NULL, so match compares
+      # them with =. It compares every other column NULL-safe: the value
+      # columns are often NULL, and with = a NULL never equals a NULL.
+      KEY_COLUMNS = %i[person_id criterion_id criterion_table criterion_domain start_date uuid window_id].freeze
+
+      def query(db)
+        rhs = right.evaluate(db)
+        rhs = rhs.from_self.select_group(*columns)
+        query = db.from(Sequel.as(left.evaluate(db), :l))
+
+        join_check = columns.map { |column| column_check(column) }.inject(&:&)
+        sub_select = rhs.from_self(alias: :r)
+                        .select(1)
+                        .where(join_check)
+
+        query.send(where_method(:where), sub_select.exists).select_all(:l)
+      end
+
+      def columns
+        @columns ||= determine_columns
+      end
+
+      # join_columns has the meaning that BinaryOperatorOperator gives it:
+      # the person (and window) columns plus the listed columns.
+      def determine_columns
+        join_columns = column_names_option(:join_columns)
+        return (matching_columns + join_columns).uniq if join_columns
+
+        only_columns = column_names_option(:only_columns)
+        except_columns = column_names_option(:except_columns)
+        columns = dynamic_columns
+        columns &= only_columns if only_columns
+        columns -= except_columns if except_columns
+        columns
+      end
+
+      def where_method(meth)
+        return meth unless invert_match
+
+        meth == :where ? :exclude : :where
+      end
+
+      # Existing statements give the string "true". The string "false" (in any
+      # case, with any spaces around it) and a blank value do not invert.
+      def invert_match
+        text = options[:invert_match].to_s.strip
+        !(text.empty? || text.casecmp?("false"))
+      end
+
+      private
+
+      def column_check(column)
+        left_column = Sequel[:l][column]
+        right_column = Sequel[:r][column]
+        return Sequel.expr([ [ left_column, right_column ] ]) if KEY_COLUMNS.include?(column)
+
+        rdbms.null_safe_equal(left_column, right_column)
+      end
+
+      # join_columns sets the whole column set, so only_columns and
+      # except_columns would have no effect next to it.
+      def additional_validation(_db, _opts = {})
+        return unless column_names_option(:join_columns)
+
+        conflicts = %i[only_columns except_columns].select { |name| column_names_option(name) }
+        add_error("conflicting options", "join_columns", *conflicts.map(&:to_s)) unless conflicts.empty?
+      end
+    end
+  end
+end

data/lib/conceptql/operators/multiple_vocabularies.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Operators
+    class MultipleVocabularies < Operator
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::CodeLister
+
+      class << self
+        def multiple_vocabularies
+          @multiple_vocabularies ||= get_multiple_vocabularies
+        end
+
+        def get_multiple_vocabularies
+          [ ConceptQL.multiple_vocabularies_file_path,
+           ConceptQL.custom_multiple_vocabularies_file_path ].select(&:exist?).map do |path|
+            CSV.foreach(path, headers: true, header_converters: :symbol).each_with_object({}) do |row, h|
+              (h[operator_symbol(row[:operator])] ||= []) << row.to_hash
+            end
+          end.inject(&:merge)
+        end
+
+        def register_many
+          multiple_vocabularies.each_key do |operator_sym|
+            register(operator_sym)
+          end
+        end
+
+        def operator_symbol(word)
+          word.gsub(/\W+/, "_").downcase
+        end
+
+        # This will override the to_metadata method and return the preferred name
+        # based on the name listed in the file.
+        #
+        # This method will be called once for each vocabulary we register
+        # for this operator
+        def to_metadata(name, opts = {})
+          h = super
+          op_info = multiple_vocabularies[name].first
+          h[:preferred_name] = op_info[:operator]
+          h[:predominant_domains] = multiple_vocabularies[name].map { |mv| mv[:domain] }.uniq.compact
+          h
+        end
+      end
+
+      register_many
+
+      desc "Selects records based on the given vocabularies."
+      argument :codes, type: :codelist
+      basic_type :selection
+      category "Select by Clinical Codes"
+      validate_no_upstreams
+      validate_at_least_one_argument
+      conceptql_spec_id "vocabulary"
+
+      def query(db)
+        # TODO: A much-more efficient method would be to find all those vocabs
+        # sharing a common table and feed them into a single single query,
+        # but I think this would require some revamping of Vocabulary, and I'm
+        # just not interested in taking that on right now.
+        vocab_ops.map { |vo| vo.evaluate(db) }.inject do |union, q|
+          union.union(q.from_self)
+        end
+      end
+
+      def domains(_db)
+        vocab_ops.map(&:domain).uniq
+      end
+
+      # Each vocabulary operator in vocab_ops has already applied the scope's
+      # windows. Applying them again to the union multiplied rows: an event in
+      # k overlapping windows came out k*k times instead of k (conceptql-7cd).
+      def skip_windows?
+        true
+      end
+
+      def source_table
+        nil
+      end
+
+      def table
+        nil
+      end
+
+      def domain
+        nil
+      end
+
+      def tables
+        []
+      end
+
+      # The per-code warnings a Vocabulary operator gives, from most to least
+      # specific. See #code_warnings.
+      BAD_FORMAT = "improperly formatted code"
+      UNKNOWN_CODE = "unknown code(s)"
+      CODE_WARNINGS = [ BAD_FORMAT, UNKNOWN_CODE ].freeze
+
+      def additional_validation(db, opts = {})
+        vocab_ops.each { |vo| vo.valid?(db, opts) }
+
+        @errors += aggregate_messages(vocab_ops.map(&:errors))
+        @warnings += aggregate_messages(vocab_ops.map { |vo| vo.warnings.reject { |w| CODE_WARNINGS.include?(w.first) } })
+        @warnings += code_warnings
+      end
+
+      # A code gets a warning only when every vocabulary rejects it, for any
+      # reason. It is an improperly formatted code only when every vocabulary
+      # says so. When at least one vocabulary accepts its format but does not
+      # know it, it is an unknown code. (12345 is badly formatted for HCPCS but
+      # a well-formed, unknown CPT4 code, so it is unknown.)
+      def code_warnings
+        verdicts = vocab_ops.map do |vo|
+          vo.warnings.each_with_object({}) do |(message, *codes), verdict|
+            codes.each { |code| verdict[code] ||= message } if CODE_WARNINGS.include?(message)
+          end
+        end
+
+        rejected = arguments.uniq.select { |code| verdicts.all? { |verdict| verdict.key?(code) } }
+        bad_format, unknown = rejected.partition { |code| verdicts.all? { |verdict| verdict[code] == BAD_FORMAT } }
+
+        [ [ BAD_FORMAT, *bad_format ], [ UNKNOWN_CODE, *unknown ] ].reject { |warning| warning.length == 1 }
+      end
+
+      def aggregate_messages(message_sets)
+        messages = message_sets.map do |message_set|
+          message_set.to_h { |message, *args| [ message, args ] }
+        end
+
+        messages_keys = messages.map(&:keys).inject(:|)
+
+        messages = messages_keys.each.with_object({}) do |key, h|
+          h[key] = messages.map { |w| w[key] || [] }.inject(:&)
+        end
+
+        messages = messages.reject { |_, v| v.empty? }
+
+        messages.map { |k, v| [ k, *v ] }
+      end
+
+      def describe_codes(db, codes)
+        codes = vocab_ops.flat_map do |vo|
+          vo.describe_codes(db, codes)
+        end
+
+        # The first vocabulary that describes a code wins, so CPT4_HCPCS adds
+        # descriptions only for codes that CPT4 and HCPCS do not know.
+        codes_with_descriptions = codes.select(&:last).uniq(&:first)
+        codes_without_descriptions = codes.reject(&:last).map(&:first)
+        codes_without_descriptions -= codes_with_descriptions.map(&:first)
+        codes_with_descriptions + codes_without_descriptions.zip([])
+      end
+
+      # An optional vocabulary (CPT4_HCPCS) has an operator only on a Database
+      # whose lexicon carries it, so it is skipped elsewhere. A required one
+      # that is missing still raises.
+      def vocab_ops
+        @vocab_ops ||= self.class.multiple_vocabularies[op_name].filter_map do |op_info|
+          name = op_info[:vocabulary_id].to_s.downcase
+          next if op_info[:optional] == "yes" && !nodifier.operators.key?(name)
+
+          nodifier.operators.fetch(name).new(nodifier, name, *arguments)
+        end
+      end
+
+      def preferred_name
+        self.class.multiple_vocabularies[op_name].first[:operator]
+      end
+    end
+  end
+end

data/lib/conceptql/operators/numeric.rb

@@ -1,10 +1,12 @@
-require_relative 'pass_thru'
+# frozen_string_literal: true
+
+require_relative "pass_thru"
 
 module ConceptQL
   module Operators
     # Represents a operator that will either:
     # - create a value_as_number value for every person in the database
-    # - change the value_as_number value for every every result passed in
+    # - change the value_as_number value for every every record passed in
     #   - either to a number
     #   - or a value from a column in the origin row
     #
@@ -12,43 +14,83 @@ module ConceptQL
     # - Either a number value or a symbol representing a column name
     # - An optional stream
     class Numeric < PassThru
-      desc <<-EOF
-Represents a operator that will either:
-- create a value_as_number value for every person in the database
-- change the value_as_number value for every every result passed in
-  - either to a number
-  - or a value from a column in the origin row
-
-Accepts two params:
-- Either a number value or a symbol representing a column name
-- An optional stream
+      register __FILE__
+
+      desc <<~EOF
+        Represents an operator that will either:
+        - Create a value_as_number value for every person in the database
+        - Change the value_as_number value for every record passed in
+          - Either to a number
+          - Or a value from a column in the origin row
+
+        Accepts two params:
+        - Either a number value or a symbol representing a column name
+        - An optional stream
       EOF
       argument :value, type: :float
       allows_one_upstream
+      validate_at_most_one_upstream
+      validate_one_argument
+      default_query_columns
+      require_column :value_as_number
 
       def query(db)
         stream.nil? ? as_criterion(db) : with_kids(db)
       end
 
-      def types
-        stream.nil? ? [:person] : super
+      def domains(db)
+        stream.nil? ? [ :person ] : super
+      end
+
+      # Without an upstream, Numeric emits exactly one row per patient via the
+      # as_criterion path (SELECT value_as_number FROM patients). The leaf
+      # default in Operator#events_per_patient assumes :multiple, which is wrong
+      # for this single-row-per-patient shape. Override here so that downstream
+      # operators (Union, etc.) can correctly conclude that a numeric leaf is
+      # single-event-per-patient.
+      def events_per_patient
+        stream.nil? ? :single : super
       end
 
       private
+
       def with_kids(db)
-        db.from(stream.evaluate(db))
-          .select(*(COLUMNS - [:value_as_number]))
-          .select_append(Sequel.lit('?', arguments.first).cast(Float).as(:value_as_number))
-          .from_self
+        dm.selectify(db.from(stream.evaluate(db)), replace: { value_as_number: first_argument })
       end
 
       def as_criterion(db)
-        db.from(select_it(db.from(:person), :person))
-          .select(*(COLUMNS - [:value_as_number]))
-          .select_append(Sequel.lit('?', arguments.first).cast(Float).as(:value_as_number))
-          .from_self
+        # Explicitly set column_family to DEFAULT_COLUMN_FAMILY here. The
+        # selectify path through `domain: :person` runs against the patients
+        # table, which doesn't have a column_family column. Without this
+        # replace, nullified_columns inserts CAST(NULL AS text) AS
+        # "column_family", and the outer evaluate -> select_it wrapper does not
+        # re-inject a value (Numeric has no source_table/table/domain methods,
+        # so determine_table returns nil and select_it's `cf =
+        # DEFAULT_COLUMN_FAMILY` branch never fires). Result: every Numeric
+        # as_criterion row carried column_family = NULL. Explicitly replacing
+        # it here keeps the fix at the same level as Episode#query, which
+        # already sets column_family in its replace hash for the same reason.
+        dm.selectify(db.from(dm.table_by_domain(:person)),
+                     domain: :person,
+                     replace: {
+                       value_as_number: first_argument,
+                       column_family: ConceptQL::Scope::DEFAULT_COLUMN_FAMILY
+                     })
+      end
+
+      def first_argument
+        value = arguments.first.to_s.strip
+        value = nil if value.blank?
+        if !value || is_a_number?(value)
+          Sequel.cast_numeric(value, Float)
+        else
+          Sequel.identifier(value)
+        end
+      end
+
+      def is_a_number?(value)
+        value =~ /\A[-+]?[0-9]*\.?[0-9]+\Z/
       end
     end
   end
 end
-

data/lib/conceptql/operators/numeric_filter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    # Filters the incoming stream of records to only those that have a
+    # value_as_number column with a value that matches the specified criteria
+    # are passed through
+    #
+    # If an record has NULL for value_as_number, it is filtered out.
+    class NumericFilter < Operator
+      register __FILE__
+
+      desc "Includes records with a value_as_number that match the given criteria."
+
+      option :greater_than_or_equal_to, type: :float
+      option :less_than_or_equal_to, type: :float
+      category "Filter Single Stream"
+      basic_type :temporal
+      allows_one_upstream
+      validate_one_upstream
+      require_column :value_as_number
+      default_query_columns
+
+      VALUE_COLUMN = Sequel[:value_as_number]
+
+      def query(db)
+        db.from(stream.evaluate(db))
+          .where(range_criteria)
+      end
+
+      private
+
+      def range_criteria
+        criteria = []
+        criteria << Sequel.~(VALUE_COLUMN => nil)
+        criteria << gte_criteria if gte.present?
+        criteria << lte_criteria if lte.present?
+        criteria.inject(&:&)
+      end
+
+      def gte
+        options[:greater_than_or_equal_to]
+      end
+
+      def lte
+        options[:less_than_or_equal_to]
+      end
+
+      def gte_literal
+        Sequel.cast_numeric(gte.to_f, Float)
+      end
+
+      def lte_literal
+        Sequel.cast_numeric(lte.to_f, Float)
+      end
+
+      def gte_criteria
+        gte_literal <= VALUE_COLUMN
+      end
+
+      def lte_criteria
+        lte_literal >= VALUE_COLUMN
+      end
+    end
+  end
+end

data/lib/conceptql/operators/occurrence.rb

@@ -1,4 +1,7 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
+require_relative "../behaviors/single_event"
 
 module ConceptQL
   module Operators
@@ -16,55 +19,173 @@ module ConceptQL
     # by start_date and then select at most one row per person, regardless
     # of how many different types of streams enter the operator
     #
-    # If two rows have the same start_date, the order of their ranking
-    # is arbitrary
+    # If two rows have the same start_date, criterion_id breaks the tie.
+    # criterion_id is unique within one criterion_table, so only rows from
+    # two different tables with the same id and start_date still rank in
+    # arbitrary order
     #
     # If we ask for the second occurrence of something and a person has only one
     # occurrence, this operator returns nothing for that person
     class Occurrence < Operator
-      preferred_name 'Nth Occurrence'
-      desc <<-EOF
-Groups all results by person, then orders by start_date, then finds the nth occurrence.
-nth occurrence can be positive or negative.
-1 => first
-2 => second
-...
--1 => last
--2 => second-to-last
-
-If two rows have the same start_date, the order of their ranking
-is arbitrary
-
-If we ask for the second occurrence of something and a person has only one
-occurrence, this operator returns nothing for that person
-      EOF
+      include ConceptQL::Behaviors::SingleEvent
+
+      register __FILE__
+
+      preferred_name "Nth Occurrence"
+
+      desc "Groups all records by person, then orders by start_date and finds the nth occurrence. Can be positive or negative, e.g 2 means 'second' and -3 means 'third from last'."
+
       argument :occurrence, type: :integer
+      category "Filter Single Stream"
+      basic_type :temporal
       allows_one_upstream
-      category %w(Temporal Occurrence)
+      validate_at_least_one_upstream
 
-      def query(db)
-        db[:occurrences]
-          .with(:occurrences,
-            stream.evaluate(db)
-              .from_self
-              .select_append { |o| o.row_number(:over, partition: :person_id, order: ordered_columns){}.as(:rn) })
-          .where(rn: occurrence.abs)
+      option :at_least, type: :string,
+                        instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+      option :within, type: :string,
+                      instructions: 'Enter a numeric value and specify "d", "m", or "y" for "days", "months", or "years". Negative numbers change dates prior to the existing date. Example: -30d = 30 days before the existing date.'
+      option :group_by_date, type: :boolean,
+                             instructions: "Choose whether to group by date when determining the nth occurrence, treating occurrences on the same day as a single occurrence"
+
+      def query_cols
+        dynamic_columns + [ :rn ]
       end
 
       def occurrence
         @occurrence ||= arguments.first
       end
 
+      def query(db)
+        ds = stream.evaluate(db)
+        additional_matching_columns = []
+        additional_matching_columns << :start_date if grouped?
+
+        ds = ds.select_append(
+          Sequel[:ROW_NUMBER].function.over(
+            partition: matching_columns_plus(*additional_matching_columns),
+            order: ordered_columns(:criterion_id)
+          ).as(:rn)
+        )
+        ds = ds.from_self
+        return ds.where(rn: occurrence.abs) unless timed? || grouped?
+
+        additional_order_columns = []
+
+        if grouped?
+          one_date_ds = ds.from_self.where(rn: 1).from_self.select_remove(:rn)
+          ds = one_date_ds.select_append(
+            Sequel[:ROW_NUMBER].function.over(
+              partition: matching_columns_plus,
+              order: ordered_columns
+            ).as(:rn)
+          ).from_self
+        end
+        return ds.where(rn: occurrence.abs) unless timed?
+
+        additional_order_columns << :criterion_id
+
+        input_name = cte_name(:occurrence_input)
+        joined_name = cte_name(:occurrence_joined)
+
+        input_ds = ds.select_append(
+          Sequel[:ROW_NUMBER].function.over(
+            partition: matching_columns_plus,
+            order: ordered_columns(*additional_order_columns, allow_reverse: false)
+          ).as(:global_rn)
+        )
+
+        first_ds = db[input_name].from_self(alias: :first)
+        first = Sequel[:first]
+        rest = Sequel[:rest]
+
+        joined_ds = first_ds.join(input_name, matching_columns_plus.map { |c| [ c, c ] }, table_alias: :rest) do
+          cond = rest[:global_rn] > first[:global_rn]
+          cond &= rest[:start_date] >= adjust_date(at_least_option, first[:end_date]) if at_least_option
+          cond &= rest[:start_date] <= adjust_date(within_option, first[:end_date]) if within_option
+          cond | { rest[:global_rn] => first[:global_rn] }
+        end
+          .select_all(:rest)
+          .select_append(first[:global_rn].as(:initial_rn))
+          .select_append(
+            Sequel[:ROW_NUMBER].function.over(
+              partition: matching_columns_plus(:global_rn).map { |c| first[c] },
+              order: ordered_columns(*additional_order_columns, qualifier: :rest)
+            ).as(:timed_rn)
+          )
+
+        db[joined_name]
+          .with(input_name, input_ds)
+          .with(joined_name, joined_ds)
+          .where(timed_rn: occurrence.abs)
+      end
+
+      def grouped?
+        options[:group_by_date]
+      end
+
+      # A blank at_least or within counts as not set. An editor can send
+      # a blank value for an empty field.
+      def timed?
+        !(at_least_option || within_option).nil?
+      end
+
+      def matching_columns_plus(*columns)
+        matching_columns + columns
+      end
+
       private
+
+      def within_option
+        return unless (v = options[:within])
+        return if v.strip.empty?
+
+        v
+      end
+
+      def at_least_option
+        return unless (v = options[:at_least])
+        return if v.strip.empty?
+
+        v
+      end
+
+      def adjust_date(adjustment, column, reverse = false)
+        adjuster = DateAdjuster.new(self, adjustment)
+        adjuster.adjust(column, reverse)
+      end
+
+      def additional_validation(_db, _opts = {})
+        if instance_of?(Occurrence)
+          validate_one_argument
+        else
+          validate_no_arguments
+        end
+      end
+
       def asc_or_desc
-        occurrence < 0 ? :desc : :asc
+        occurrence.negative? ? :desc : :asc
+      end
+
+      def qualify_with(qualifier, col)
+        if qualifier
+          Sequel.qualify(qualifier, col)
+        else
+          col
+        end
       end
 
-      def ordered_columns
-        ordered_columns = [Sequel.send(asc_or_desc, :start_date)]
-        ordered_columns += [:criterion_type, :criterion_id]
+      def ordered_columns(*columns, qualifier: nil, allow_reverse: true)
+        start_date = qualify_with(qualifier, :start_date)
+        start_date = Sequel.send(asc_or_desc, start_date) if allow_reverse
+        order = [ start_date ]
+        order += columns.map do |c|
+          c = qualify_with(qualifier, c)
+          c = Sequel.send(asc_or_desc, c) if allow_reverse
+          c
+        end
+        order
       end
     end
   end
 end
-