conceptql 0.2.0 → 0.4.0

data/lib/conceptql/behaviors/metadatable.rb

@@ -1,97 +1,250 @@
-require 'facets/kernel/meta_def'
-require 'facets/string/snakecase'
+# frozen_string_literal: true
 
-module Metadatable
-  def preferred_name(value = nil)
-    return @preferred_name unless value
-    @preferred_name = value
-  end
+module ConceptQL
+  module Metadatable
+    def preferred_name(value = nil)
+      return @preferred_name unless value
 
-  def desc(value = nil)
-    return @desc unless value
-    @desc = value
-  end
+      @preferred_name = value
+    end
 
-  def predominant_types(*values)
-    return @predominant_types if values.empty?
-    @predominant_types = values
-  end
+    def aliases(value = nil)
+      return @aliases unless value
 
-  def argument(name, options = {})
-    (@arguments ||= [])
-    @arguments << [name, options]
-  end
+      @aliases = value
+    end
 
-  def option(name, options = {})
-    @options ||= {}
-    @options[name] = options
-  end
+    def desc(value = nil)
+      return @desc unless value
+
+      @desc = value
+    end
+
+    def short_name(value = nil)
+      return @short_name unless value
+
+      @short_name = value
+    end
+
+    def long_name(value = nil)
+      return @long_name unless value
+
+      @long_name = value
+    end
+
+    def conceptql_spec_id(value = nil)
+      return @conceptql_spec_id unless value
+
+      @conceptql_spec_id = value
+    end
+
+    def predominant_domains(*values)
+      return @predominant_domains if values.empty?
+
+      @predominant_domains = values
+    end
+
+    def argument(name, options = {})
+      (@arguments ||= [])
+      @arguments << [ name, auto_label(name, options) ]
+    end
+
+    def option(name, options = {})
+      @options ||= {}
+      @options[name] = auto_label(name, options)
+    end
+
+    # Takes an option that a subclass inherited out of its metadata
+    def remove_option(name)
+      @options.delete(name)
+    end
+
+    def auto_label(name, opts = {})
+      return opts if opts[:label]
+      return opts.merge(label: name.to_s.split("_").map(&:capitalize).join(" ")) unless opts[:type] == :codelist
 
-  def types(*type_list)
-    @types = type_list
-    define_method(:types) do
-      type_list
+      opts.merge(label: "#{pref_name} Codes")
     end
-    if type_list.length == 1
-      define_method(:type) do
-        type_list.first
+
+    def domains(*domain_list)
+      @domains = domain_list
+      define_method(:domains) do |_db|
+        domain_list
+      end
+      return unless domain_list.length == 1
+
+      define_method(:domain) do
+        domain_list.first
       end
     end
-  end
 
-  def allows_many_upstreams
-    @max_upstreams = 99
-  end
+    def basic_type(value = nil)
+      return @basic_type unless value
 
-  def allows_one_upstream
-    @max_upstreams = 1
-  end
+      @basic_type = value
+    end
 
-  def just_class_name
-    self.to_s.split('::').last
-  end
+    def allows_many_upstreams
+      @max_upstreams = 99
+    end
 
-  def humanized_class_name
-    just_class_name.gsub(/([A-Z])/, ' \1').lstrip
-  end
+    def allows_one_upstream
+      @max_upstreams = 1
+    end
 
-  def category(category)
-    (@categories ||= [])
-    @categories << Array(category)
-  end
+    def just_class_name
+      to_s.split("::").last
+    end
 
-  def reset_categories
-    @categories = []
-  end
+    def humanized_class_name
+      just_class_name.gsub(/([A-Z])/, ' \1').lstrip
+    end
 
-  def inherited(upstream)
-    (@options || {}).each do |name, opt|
-      upstream.option name, opt
+    def category(category)
+      (@categories ||= [])
+      @categories << Array(category)
     end
 
-    (@categories || []).each do |cat|
-      upstream.category cat
+    def reset_categories
+      @categories = []
     end
 
-    case @max_upstreams
-    when 1
-      upstream.allows_one_upstream
-    when 99
-      upstream.allows_many_upstreams
+    def inherited(upstream)
+      super
+      (@options || {}).each do |name, opt|
+        upstream.option name, opt
+      end
+
+      (@categories || []).each do |cat|
+        upstream.category cat
+      end
+
+      upstream.basic_type @basic_type
+
+      case @max_upstreams
+      when 1
+        upstream.allows_one_upstream
+      when 99
+        upstream.allows_many_upstreams
+      end
+    end
+
+    def pref_name
+      @preferred_name || humanized_class_name
+    end
+
+    def spec_id
+      "#{@conceptql_spec_id || pref_name} operator".gsub(/\s+/, "-").downcase
+    end
+
+    def to_metadata(name, opts = {})
+      derive_metadata_from_validations
+      warn_about_missing_metadata if opts[:warn]
+      {
+        name: name,
+        preferred_name: pref_name,
+        short_name: short_name,
+        long_name: long_name,
+        operation: just_class_name.underscore,
+        min_upstreams: @max_upstreams || 0,
+        max_upstreams: @max_upstreams || 0,
+        arguments: @arguments || [],
+        options: process_options(@options),
+        predominant_domains: @domains || @predominant_domains || [],
+        desc: get_desc,
+        categories: @categories || [],
+        basic_type: @basic_type,
+        deprecated: nil, # kept for metadata consumers; no operator is deprecated
+        arity: arity,
+        aliases: @aliases,
+        spec_id: spec_id
+      }
+    end
+
+    def arity
+      case basic_type
+      when :set
+        -1
+      when :temporal, :cast
+        1
+      when :selection, nil
+        0
+      when :filter
+        2
+      else
+        raise "#{name} has unknown basic_type: #{basic_type.pretty_inspect}"
+      end
+    end
+
+    def get_desc
+      @get_desc ||= standard_description
     end
-  end
 
-  def to_metadata
-    {
-      preferred_name: @preferred_name || humanized_class_name,
-      operation: just_class_name.snakecase,
-      max_upstreams: @max_upstreams || 0,
-      arguments: @arguments || [],
-      options: @options || {},
-      predominant_types: @types || @predominant_types || [],
-      desc: @desc,
-      categories: @categories || []
-    }
+    def process_options(options)
+      return {} if options.nil? || options.empty?
+
+      # Build new hashes: the option hashes can be shared with a parent class
+      options.transform_values do |v|
+        v[:options].respond_to?(:call) ? v.merge(options: v[:options].call) : v
+      end
+    end
+
+    def warn_about_missing_metadata
+      missing = []
+      missing << :categories if (@categories || []).empty?
+      missing << :desc if get_desc.empty?
+      missing << :basic_type unless @basic_type
+      puts "#{just_class_name} is missing #{missing.join(', ')}" unless missing.empty?
+    end
+
+    # Each validation is [meth, *args]. Every argument reaches the hook, so
+    # validate_required_options :left, :right marks both options.
+    def derive_metadata_from_validations
+      @validations.each do |meth, *args|
+        hook = "#{meth}_to_metadata"
+        send(hook, *args) if respond_to?(hook)
+      end
+    end
+
+    def validate_no_upstreams_to_metadata(*_args)
+      @min_upstreams = 0
+      @max_upstreams = 0
+    end
+
+    def validate_one_upstream_to_metadata(*_args)
+      @min_upstreams = 1
+      @max_upstreams = 1
+    end
+
+    def validate_at_least_one_upstream_to_metadata(*_args)
+      @min_upstreams = 1
+      @max_upstreams = 99
+    end
+
+    def validate_at_most_one_upstream_to_metadata(*_args)
+      @min_upstreams = 0
+      @max_upstreams = 1
+    end
+
+    def validate_no_arguments_to_metadata(*_args)
+      @arguments = []
+    end
+
+    # A subclass shares the option hashes that it inherits (see #inherited),
+    # so mark a copy. Writing into the shared hash would mark the option as
+    # required for the parent class too.
+    def validate_required_options_to_metadata(*names)
+      names.each do |name|
+        @options[name] = @options.fetch(name).merge(required: true)
+      end
+    end
+
+    def standard_description
+      "Selects records based on the #{pref_name} vocabulary."
+    end
+
+    def no_desc
+      desc ""
+    end
   end
 end
-

data/lib/conceptql/behaviors/provenanceable.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Provenanceable
+    def self.included(base)
+      base.require_column :file_provenance_type
+      base.require_column :code_provenance_type
+    end
+
+    def prov_of(db, ancestors)
+      dm.descendants_of(db,
+                        dm.concept_ids(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten,
+                                       ancestors))
+    end
+
+    def build_where_from_codes(db, codes)
+      Sequel.|(
+        { file_provenance_type: prov_of(db, codes) }, { code_provenance_type: prov_of(db, codes) }
+      )
+    end
+
+    def find_bad_keywords(db, codes)
+      codes - dm.concept_codes(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten)
+    end
+  end
+end

data/lib/conceptql/behaviors/single_event.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    module SingleEvent
+      def events_per_patient
+        :single
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/timeless.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    module Timeless
+      def timeless?
+        true
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/utilizable.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module ConceptQL
+  module Behaviors
+    module Utilizable
+      extend ActiveSupport::Concern
+      include ConceptQL::Behaviors::Windowable
+
+      included do
+        domains :condition_occurrence
+        category "Select by Property"
+        basic_type :selection
+        validate_no_upstreams
+        validate_no_arguments
+
+        require_column :admission_date
+        require_column :discharge_date
+        require_column :length_of_stay
+        require_column :admission_source_value
+        require_column :admission_source_description
+        require_column :discharge_location_source_value
+        require_column :discharge_location_source_description
+        require_column :primary_concept_id
+      end
+
+      def column_family
+        "inpatient"
+      end
+
+      # The operators that include this register for gdm and gdm_wide only.
+      def query(db)
+        gdm_it(db)
+      end
+
+      def collection_type
+        raise NotImplementedError
+      end
+
+      # With the Database's share_utilization option, the list is read from a
+      # table that every statement on the Database shares. The WITH entry only
+      # carries the list to Scope#with_ctes, which takes it out of the SQL and
+      # creates the table when the query first reads rows.
+      def evaluate(db)
+        cdb = nodifier.cdb
+        return super unless cdb.opts[:share_utilization]
+
+        list = select_it(query(db))
+        name = cdb.shared_utilization_table(list, scope)
+        db[name].with(name, list, Scope::SHARED_TABLE => true)
+      end
+
+      def gdm_it(db)
+        source_type_id = dm.concept_ids(db, dm.file_provenance_types_vocab, collection_type)
+        all_source_type_ids = dm.descendants_of(db, source_type_id)
+        primary_id = dm.concept_ids(db, dm.code_provenance_types_vocab, "primary")
+        all_primary_ids = dm.descendants_of(db, primary_id)
+        # condition_vocabularies = dm.vocabularies_query.where(domain: 'condition_occurrence').select_map(:id)
+
+        return gdm_wide_it(db, all_primary_ids, all_source_type_ids) if wide?
+
+        # Get primary diagnosis codes
+        primary_concepts = rdbms.primary_concepts(db, all_primary_ids)
+
+        relevant_contexts = db[Sequel[:contexts].as(:cn)]
+                            .where(Sequel[:cn][:source_type_concept_id] => all_source_type_ids)
+                            .select(:collection_id)
+
+        db[:collections].from_self(alias: :cl)
+                        .then { |cl| restrict_to_window_persons(cl, Sequel[:cl][:patient_id]) }
+                        .join(:admission_details, { Sequel[:ad][:id] => Sequel[:cl][:admission_detail_id] }, table_alias: :ad)
+                        .left_join(dm.name_concepts_table(db), { Sequel[:ad][:admit_source_concept_id] => Sequel[:asc][:id] }, table_alias: :asc)
+                        .left_join(dm.name_concepts_table(db), { Sequel[:ad][:discharge_location_concept_id] => Sequel[:dlc][:id] }, table_alias: :dlc)
+                        .left_join(primary_concepts, { Sequel[:pcon][:collection_id] => Sequel[:cl][:id] }, table_alias: :pcon)
+                        .where(Sequel[:cl][:id] => relevant_contexts)
+                        .select(
+                          Sequel[:cl][:start_date].as(:collection_start_date),
+                          Sequel[:cl][:end_date].as(:collection_end_date),
+                          Sequel[:ad][:admission_date].as(:admission_date),
+                          Sequel[:ad][:discharge_date].as(:discharge_date),
+                          Sequel[:asc][:concept_code].as(:admission_source_value),
+                          Sequel[:asc][:concept_text].as(:admission_source_description),
+                          Sequel[:dlc][:concept_code].as(:discharge_location_source_value),
+                          Sequel[:dlc][:concept_text].as(:discharge_location_source_description),
+                          Sequel[:pcon][:concept_id].as(:primary_concept_id),
+                          Sequel[:pcon][:concept_code].as(:source_value),
+                          Sequel[:pcon][:vocabulary_id].as(:source_vocabulary_id),
+                          Sequel[:cl][:patient_id].as(:patient_id),
+                          Sequel[:cl][:id].as(:collection_id)
+                        )
+                        .select_append(Sequel[:ROW_NUMBER].function.over(
+                          partition: [ :admission_detail_id ],
+                          order: [
+                            Sequel[:pcon][:is_primary].desc,
+                            Sequel[:cl][:id],
+                            Sequel[:pcon][:concept_code]
+                          ]
+                        ).as(:nummy))
+                        .from_self
+                        .where(nummy: 1)
+      end
+
+      def gdm_wide_it(db, all_primary_ids, all_source_type_ids)
+        # Emit `CASE WHEN provenance_concept_id IN (...) THEN 1 ELSE 0 END`.
+        # The previous multi-WHEN form (`Sequel.case({{prov=>id1}=>1, {prov=>id2}=>1, ...}, 0)`)
+        # is mis-evaluated by Spark 3.3.2 when its result feeds the outer
+        # `CASE is_primary WHEN 1 THEN col ELSE NULL END` below. See conceptql-pk7.
+        is_primary =
+          if all_primary_ids.empty?
+            Sequel[0]
+          else
+            Sequel.case(
+              [ [ { Sequel[:provenance_concept_id] => all_primary_ids }, 1 ] ],
+              0
+            )
+          end
+
+        db[:observations]
+          .then { |observations| restrict_to_window_persons(observations, :patient_id) }
+          .where(source_type_concept_id: all_source_type_ids)
+          .exclude(admit_admission_date: nil)
+          .select_append(
+            is_primary.as(:is_primary)
+          )
+          .from_self
+          # One admission has many rows with the same concept on different
+          # line dates. The unique id ends the ORDER BY so the kept row is the
+          # same on every run, and admission_end_date takes the latest end_date
+          # over all of the admission's rows, not the kept row's. See conceptql-evk.
+          .select_append(
+            Sequel[:ROW_NUMBER].function.over(
+              partition: [ :admission_detail_id ],
+              order: [ Sequel[:is_primary].desc, Sequel[:collection_id], Sequel[:clinical_code_concept_id], Sequel[:id] ]
+            ).as(:nummy),
+            Sequel.function(:max, :end_date).over(partition: [ :admission_detail_id ]).as(:admission_end_date)
+          )
+          .from_self
+          .where(nummy: 1)
+          .from_self
+          .left_join(dm.name_concepts_table(db), { Sequel[:admit_source_concept_id] => Sequel[:asc][:id] }, table_alias: :asc)
+          .left_join(dm.name_concepts_table(db), { Sequel[:discharge_location_concept_id] => Sequel[:dlc][:id] }, table_alias: :dlc)
+          .select_append(
+            Sequel[:asc][:concept_code].as(:admission_source_value),
+            Sequel[:asc][:concept_text].as(:admission_source_description),
+            Sequel[:dlc][:concept_code].as(:discharge_location_source_value),
+            Sequel[:dlc][:concept_text].as(:discharge_location_source_description),
+            Sequel.case({ 1 => :clinical_code_concept_id }, nil, :is_primary).as(:primary_concept_id),
+            Sequel.case({ 1 => :clinical_code_source_value }, nil, :is_primary).as(:source_value),
+            Sequel.case({ 1 => :clinical_code_vocabulary_id }, nil, :is_primary).as(:source_vocabulary_id)
+          )
+          .from_self
+      end
+
+      # Keep only the persons that the scope's table windows cover, before the
+      # ROW_NUMBER over admission_detail_id. Without this, Spark numbers every
+      # inpatient row in the database and applies the window join afterwards.
+      # Output is unchanged because one admission never belongs to two persons
+      # (Ryan, 2026-10-14), so each ROW_NUMBER partition is kept or dropped whole.
+      def restrict_to_window_persons(ds, person_column)
+        return ds if skip_windows?
+
+        scope.windows.grep(Window::Table).inject(ds) do |restricted, window|
+          restricted.where(person_column => window.person_ids(restricted.db))
+        end
+      end
+
+      def table
+        :collections
+      end
+
+      def override_columns
+        return unless gdm?
+
+        {
+          length_of_stay: (rdbms.days_between(Sequel[:admission_date],
+                                              discharge_date_column) + 1).as(:length_of_stay),
+          criterion_table: Sequel.cast_string("collections").as(:criterion_table),
+          criterion_domain: Sequel.cast_string("condition_occurrence").as(:criterion_domain),
+          admission_source_value: Sequel[:admission_source_value],
+          admission_source_description: Sequel[:admission_source_description],
+          discharge_location_source_value: Sequel[:discharge_location_source_value],
+          discharge_location_source_description: Sequel[:discharge_location_source_description],
+          admission_date: Sequel[:admission_date],
+          discharge_date: discharge_date_column.as(:discharge_date),
+          person_id: Sequel[:patient_id].as(:person_id),
+          criterion_id: Sequel[:collection_id].as(:criterion_id),
+          start_date: Sequel[:collection_start_date].as(:start_date),
+          end_date: Sequel[:collection_end_date].as(:end_date),
+          primary_concept_id: Sequel[:primary_concept_id],
+          source_value: Sequel[:source_value].as(:source_value),
+          source_vocabulary_id: Sequel[:source_vocabulary_id].as(:source_vocabulary_id)
+        }
+      end
+
+      # gdm_wide's AdmissionDateQueryModifier falls back to the kept row's own
+      # end_date when the admission has no discharge date. An admission falls
+      # back to its latest end_date instead (Ryan, 2026-10-16). See conceptql-evk.
+      def discharge_date_column
+        return Sequel[:discharge_date] unless wide?
+
+        Sequel.function(:coalesce, :admit_discharge_date, :admission_end_date, :discharge_date)
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/windowable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    # Applies scope windows to Operators
+    module Windowable
+      def select_it(query, opts = {})
+        ds = super
+        return ds if skip_windows?
+
+        scope.windows.each do |window|
+          ds = window.call(self, ds, timeless: timeless?)
+        end
+
+        ds
+      end
+
+      def skip_windows?
+        false
+      end
+
+      def timeless?
+        false
+      end
+    end
+  end
+end