conceptql 0.3.0 → 0.4.0

data/lib/conceptql/operators/episode.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require_relative "pass_thru"
+
+module ConceptQL
+  module Operators
+    class Episode < PassThru
+      register __FILE__
+
+      desc "Groups each person's records into episodes, allowing for a gap between the end_date of one record and the start_date of the next."
+
+      allows_one_upstream
+      validate_one_upstream
+      option :gap_of, type: :integer,
+                      instructions: "Allowed gap in days between end date of one event and start date of the next to consider them the same episode"
+      category "Modify Data"
+      basic_type :temporal
+
+      def table
+        nil
+      end
+
+      # Episode strips source_vocabulary_id from its output rows (the SQL emits
+      # NULL for that column), so downstream consumers can't tell which vocab
+      # the data originated from. The default introspection methods on Operator
+      # walk upstream and report whatever vocabularies were upstream of Episode,
+      # which misrepresents what Episode actually emits. Override both methods
+      # to reflect that Episode erases vocabulary metadata.
+      def vocabularies
+        []
+      end
+
+      def multiple_vocabularies
+        false
+      end
+
+      def query(db)
+        ds = unioned(db)
+
+        partition_vars
+
+        dt1 = ds.select(*partition_vars).select_append(
+          :start_date,
+          :end_date,
+          Sequel[:start_date].as(:episode_start_date),
+          Sequel.function(:max,
+                          Sequel.function(:coalesce, :end_date,
+                                          :start_date)).over(partition: partition_vars, order: %i[start_date
+                                                                                                  end_date]).as(:episode_end_date)
+        )
+
+        # Get allowed episode gap
+        gap_of = get_episode_gap
+
+        gap_check = date_adjust_add(db,
+                                    Sequel.function(:lag, :episode_end_date).over(partition: partition_vars, order: %i[start_date end_date]), gap_of, "days")
+
+        cond_gap_lt_start = (gap_check < Sequel[:episode_end_date]) | (nil)
+
+        dt2 = dt1.from_self.select_append(
+          gap_of.as(:gap_of),
+          cond_gap_lt_start.as(:step) # True if new episode else null
+        )
+
+        tmp_episode_summary = dt2.from_self.select_append(
+          (Sequel.function(:count, :step).over(partition: partition_vars,
+                                               order: %i[start_date
+                                                         end_date]) + 1).as(:episode)
+        )
+
+        ids_plus_episode = (partition_vars + [ :episode ])
+
+        # Get last dispensing in an episodes gap and
+        last_dispensing = tmp_episode_summary.from_self.select(*ids_plus_episode)
+                                             .select_append(
+                                               :gap_of,
+                                               rdbms.datediff(Sequel[:episode_start_date],
+                                                              Sequel.function(:lag, :episode_end_date).over(
+                                                                partition: partition_vars + [ :episode ], order: :start_date
+                                                              )).as(:LEpisodeGap),
+                                               Sequel.function(:row_number).over(partition: ids_plus_episode,
+                                                                                 order: [ Sequel.desc(:start_date) ]).as(:event_number)
+                                             )
+
+        last_dispensing = last_dispensing.from_self.where(event_number: 1)
+
+        join_hash = ids_plus_episode.map { |c| [ c, c ] }.to_h
+
+        episode_summary = tmp_episode_summary.from_self(alias: :e).join(last_dispensing, join_hash,
+                                                                        table_alias: :o)
+        e = Sequel[:e]
+        Sequel[:o]
+
+        grp_cols = ids_plus_episode.map { |c| e[c] }
+
+        episode_summary = episode_summary.select_group(*grp_cols).select_append(
+          Sequel.function(:min, :episode_start_date).as(:start_date),
+          Sequel.function(:max, :episode_end_date).as(:end_date)
+        )
+
+        # Episodes streams return null for criterion_id,table, and domain which messes up uuid generation so we had to add constants to these values
+        episode_summary = episode_summary.from_self.select_append(
+          Sequel[0].cast_numeric.as(:criterion_id),
+          Sequel.cast_string("episode").as(:criterion_table),
+          Sequel.cast_string("episode").as(:criterion_domain)
+        )
+
+        episode_query_cols = partition_vars + %i[start_date end_date criterion_id criterion_table
+                                                 criterion_domain]
+        dm.selectify(episode_summary.from_self, { query_columns: episode_query_cols.zip(episode_query_cols).to_h,
+                                                  replace: { column_family: ConceptQL::Scope::DEFAULT_COLUMN_FAMILY } })
+      end
+
+      def get_episode_gap
+        # If Create treatment episodes get episode gap otherwise allow only 0 gap
+        return Sequel[options[:gap_of].to_i] unless options[:gap_of].nil?
+
+        Sequel[0]
+      end
+
+      def unioned(db)
+        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
+          uni.union(q)
+        end
+      end
+
+      def partition_vars
+        matching_columns
+      end
+
+      ##
+      # Adds `by` units of `timeframe` (days, months, etc) to the date `from`.
+      # PostgreSQL and DuckDB multiply a one-unit INTERVAL; other databases
+      # (Spark) use the INTERVAL n unit form.
+      #
+
+      def date_adjust_add(db, from, by, timeframe)
+        if %i[postgres duckdb].include?(db.database_type)
+          Sequel.cast(from + Sequel.lit("(? * INTERVAL '1' ?)", by, Sequel.lit(timeframe.sub(/s\z/, ""))), Date)
+        else
+          from + Sequel.lit("INTERVAL ? ?", by, Sequel.lit(timeframe))
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/operators/equal.rb

@@ -1,15 +1,18 @@
-require_relative 'temporal_operator'
+# frozen_string_literal: true
+
+require_relative "temporal_operator"
 
 module ConceptQL
   module Operators
     class Equal < TemporalOperator
       register __FILE__
 
-      desc 'If a result in the left hand results (LHR) has the same value_as_number as a result in the right hand results (RHR), it is passed through.'
+      desc "For each person, passes along left hand records that have the same value_as_number as a right hand record."
+
       require_column :value_as_number
 
       def where_clause
-        { r__value_as_number: :l__value_as_number }
+        { Sequel[:r][:value_as_number] => Sequel[:l][:value_as_number] }
       end
     end
   end

data/lib/conceptql/operators/ethnicity.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Ethnicity < Operator
+      register __FILE__
+
+      desc "Selects person records that match the given set of Ethnicity codes."
+      domains :person
+      category "Select by Property"
+      basic_type :selection
+      argument :ethnicity, type: :string, options: [ "Hispanic or Latino", "Not Hispanic or Latino" ]
+      validate_no_upstreams
+      validate_at_least_one_argument
+
+      def query_cols
+        table_columns(table)
+      end
+
+      def table
+        source_table
+      end
+
+      def source_table
+        dm.person_table
+      end
+
+      def query(db)
+        db.from(source_table)
+          .where(ethnicity_concept_id: ethnicity_concept_ids)
+      end
+
+      def ethnicity_concept_ids
+        arguments.map do |words|
+          case words
+          when /^Not Hispanic/i
+            38_003_564
+          when /^Hispanic/i
+            38_003_563
+          when /unknown/i
+            [ 0, nil ]
+          end
+        end.flatten
+      end
+    end
+  end
+end

data/lib/conceptql/operators/except.rb

@@ -1,33 +1,28 @@
-require_relative 'binary_operator_operator'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
 
 module ConceptQL
   module Operators
     class Except < BinaryOperatorOperator
       register __FILE__
 
-      desc 'If a result in the left hand results (LHR) appears in the right hand results (RHR), it is removed from the output result set.'
+      desc "Removes all incoming records that appear in the right hand set of records."
       default_query_columns
 
       def query(db)
-        if ignore_dates?
-          query = db.from(Sequel.as(left.evaluate(db), :l))
-            .left_join(Sequel.as(right.evaluate(db), :r), l__criterion_id: :r__criterion_id, l__criterion_domain: :r__criterion_domain)
-            .where(r__criterion_id: nil)
-            .select_all(:l)
-          db.from(query)
-        else
-          lquery = left.evaluate(db)
-          rquery = right.evaluate(db)
-
-          # Set columns so that impala's EXCEPT emulation doesn't use a query to determine them
-          lquery.instance_variable_set(:@columns, query_cols)
-          rquery.instance_variable_set(:@columns, query_cols)
+        cols_to_compare = %i[
+          criterion_id
+          criterion_table
+        ]
+        cols_to_compare += %i[start_date end_date] unless ignore_dates?
+        join_cols = cols_to_compare.zip(cols_to_compare).to_h
 
-          if impala?
-            lquery = lquery.except_strategy(:not_exists, :person_id, :criterion_id, :criterion_domain)
-          end
-          lquery.except(rquery)
-        end
+        query = db.from(Sequel.as(left.evaluate(db), :l))
+                  .left_join(Sequel.as(right.evaluate(db), :r), join_cols)
+                  .where(Sequel[:r][:criterion_id] => nil)
+                  .select_all(:l)
+        db.from(query)
       end
 
       private

data/lib/conceptql/operators/filter.rb

@@ -1,24 +1,36 @@
-require_relative 'binary_operator_operator'
+# frozen_string_literal: true
+
+require_relative "binary_operator_operator"
 
 module ConceptQL
   module Operators
     class Filter < BinaryOperatorOperator
       register __FILE__
 
-      desc 'If a result in the left hand results (LHR) has a corresponding result in the right hand results (RHR) with the same person, criterion_id, and criterion_domain, it is passed through.'
+      desc "Keeps left hand records that share the same person, criterion_id, and criterion_domain with a right hand side record."
       default_query_columns
 
       def query(db)
         rhs = right.evaluate(db)
-        rhs = rhs.from_self.select_group(:person_id, :criterion_id, :criterion_domain)
+        rhs = rhs.from_self.select_group(*columns)
         query = db.from(Sequel.as(left.evaluate(db), :l))
-        query = query
-          .left_join(Sequel.as(rhs, :r), l__person_id: :r__person_id, l__criterion_id: :r__criterion_id, l__criterion_domain: :r__criterion_domain)
-          .exclude(r__criterion_id: nil)
-          .select_all(:l)
+        query = semi_or_inner_join(query, rhs, join_columns)
         db.from(query)
       end
+
+      def columns
+        @columns ||= determine_columns
+      end
+
+      def join_columns
+        columns.map { |c| [ Sequel[:l][c], Sequel[:r][c] ] }
+      end
+
+      def determine_columns
+        columns = %w[person_id criterion_id criterion_domain]
+        columns += %w[start_date end_date] unless options[:ignore_dates]
+        columns.map(&:to_sym)
+      end
     end
   end
 end
-

data/lib/conceptql/operators/first.rb

@@ -1,4 +1,6 @@
-require_relative 'occurrence'
+# frozen_string_literal: true
+
+require_relative "occurrence"
 
 module ConceptQL
   module Operators
@@ -17,12 +19,21 @@ module ConceptQL
     class First < Occurrence
       register __FILE__
 
-      desc 'Only passes through the result with the earliest start_date per person.  If more than one result qualifies, the result is arbitrarily chosen.'
+      desc "For each person, passes along the record with the earliest start_date."
+
+      # With either option, Occurrence's timed query returns every upstream
+      # row instead of one row per person
+      unsupported_options :at_least, :within
 
       def occurrence
         1
       end
+
+      # A non-blank at_least or within fails validation, but the statement
+      # still builds SQL. That SQL stays the plain query, not the timed one.
+      def timed?
+        false
+      end
     end
   end
 end
-

data/lib/conceptql/operators/from.rb

@@ -1,6 +1,11 @@
+# frozen_string_literal: true
+
 module ConceptQL
   module Operators
     class From < Operator
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
       register __FILE__
       basic_type :selection
       no_desc
@@ -10,32 +15,48 @@ module ConceptQL
       validate_one_argument
 
       def query(db)
-        db.from(table)
+        db.from(table_name)
       end
 
-      def domains(db)
+      def domains(_db)
         doms = options[:domains]
         if doms.nil? || doms.empty?
-          if TABLE_COLUMNS.has_key?(table)
-            [table]
+          if dm.schema.key?(table_name)
+            [ table_name ]
           else
-            [:invalid]
+            [ :invalid ]
           end
         else
           doms.map(&:to_sym)
         end
       end
 
-      def table
-        values.first.to_sym rescue nil
+      def table_name
+        return @table_name if @table_name
+
+        name = values.first
+        if name.is_a?(String)
+          table, column = name.split("__", 2)
+          name = if column
+                   Sequel.qualify(table, column)
+          else
+                   name.to_sym
+          end
+        end
+        @table_name = name
+      end
+
+      def required_columns
+        override_columns.keys
       end
 
       def query_cols
-        cols = options[:query_cols]
-        if cols.nil? || cols.empty?
-          cols = table_columns(table) rescue dynamic_columns
-        end
-        cols
+        required_columns
+      end
+
+      def override_columns
+        cols = (options[:query_cols] || dynamic_columns).map(&:to_sym)
+        Hash[cols.zip(cols)]
       end
     end
   end

data/lib/conceptql/operators/gender.rb

@@ -1,33 +1,49 @@
-require_relative 'operator'
+# frozen_string_literal: true
+
+require_relative "operator"
 
 module ConceptQL
   module Operators
     class Gender < Operator
       register __FILE__
 
-      desc 'Returns all person records that match the selected gender.'
-      argument :gender, type: :string, options: ['Male', 'Female']
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      desc "Selects all person records that match the selected gender."
+      argument :gender, type: :string, options: %w[Male Female Unknown]
       domains :person
       category "Select by Property"
       basic_type :selection
-      query_columns :person
       validate_no_upstreams
       validate_at_least_one_argument
 
       def query(db)
-        gender_concept_ids = arguments.map do |value|
-          case value.to_s
+        clauses = arguments.map do |arg|
+          case arg.to_s
           when /^m/i
-            8507
+            Sequel.expr(gender_concept_id: male_ids(db))
           when /^f/i
-            8532
+            Sequel.expr(gender_concept_id: female_ids(db))
           else
-            value.to_i
+            Sequel.expr(gender_concept_id: nil) | Sequel.~(gender_concept_id: male_ids(db) + female_ids(db))
           end
         end
 
-        db.from(:person)
-          .where(gender_concept_id: gender_concept_ids)
+        db.from(table)
+          .where(clauses.inject(&:|))
+      end
+
+      def table
+        dm.table_by_domain(:person)
+      end
+
+      def male_ids(db)
+        dm.related_concept_ids(db, 8507)
+      end
+
+      def female_ids(db)
+        dm.related_concept_ids(db, 8532)
       end
     end
   end

data/lib/conceptql/operators/hospice.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Hospice < Operator
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
+      include ConceptQL::Behaviors::Utilizable
+
+      desc "Selects admission records of type 'hospice'"
+
+      def collection_type
+        "hospice"
+      end
+    end
+  end
+end

data/lib/conceptql/operators/hospitalization.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class Hospitalization < Operator
+      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
+      register __FILE__, :gdm, :gdm_wide
+      include ConceptQL::Behaviors::Utilizable
+
+      desc "Selects admission records of type 'inpatient'"
+
+      def collection_type
+        "inpatient"
+      end
+    end
+  end
+end

data/lib/conceptql/operators/information_periods.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "operator"
+
+module ConceptQL
+  module Operators
+    class InformationPeriods < Operator
+      register __FILE__
+
+      include ConceptQL::Behaviors::Windowable
+      include ConceptQL::Behaviors::Timeless
+
+      desc "Generates all observation_period records, or, if fed a stream, fetches all observation_period records for the people represented in the incoming record set."
+      domains :observation_period
+      category "Get Related Data"
+      basic_type :selection
+      validate_no_upstreams
+
+      def query(db)
+        ds = db[source_table]
+        ds = ds.where(where_clause) if where_clause
+        ds
+      end
+
+      def source_table
+        tab = dm.table_by_domain(domain)
+        tab = :payer_plan_period if tab == :observation_period && arguments.present?
+        tab
+      end
+
+      def where_clause
+        dm.information_period_where_clause(arguments)
+      end
+    end
+  end
+end

data/lib/conceptql/operators/invalid.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ConceptQL
   module Operators
     class Invalid < Operator
@@ -15,7 +17,7 @@ module ConceptQL
         @operator_name ||= arguments.shift.to_s || "invalid"
       end
 
-      def query(db)
+      def query(_db)
         raise "Invalid#query called.  #{errors}"
       end
 
@@ -27,10 +29,7 @@ module ConceptQL
         super
       end
 
-      def validate(db, opts = {})
-        add_error(*@errors)
-      end
+      def validate(db, opts = {}); end
     end
   end
 end
-

data/lib/conceptql/operators/last.rb

@@ -1,4 +1,6 @@
-require_relative 'occurrence'
+# frozen_string_literal: true
+
+require_relative "occurrence"
 
 module ConceptQL
   module Operators
@@ -17,12 +19,21 @@ module ConceptQL
     class Last < Occurrence
       register __FILE__
 
-      desc 'Only passes through the result with the most recent start_date per person.  If more than one result qualifies, the result is arbitrarily chosen.'
+      desc "For each person, passes along the record with the most recent start_date."
+
+      # With either option, Occurrence's timed query returns every upstream
+      # row instead of one row per person
+      unsupported_options :at_least, :within
 
       def occurrence
         -1
       end
+
+      # A non-blank at_least or within fails validation, but the statement
+      # still builds SQL. That SQL stays the plain query, not the timed one.
+      def timed?
+        false
+      end
     end
   end
 end
-