conceptql 0.4.1 → 0.6.0

data/lib/conceptql/vocabularies/formats.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "csv"
+require_relative "../paths"
+
+module ConceptQL
+  module Vocabularies
+    # The code formats that a Vocabulary operator checks its codes against.
+    # They come from config/vocabulary_formats.tsv, a pinned copy of the
+    # published table of the public repository
+    # outcomesinsights/vocabulary_formats (bead conceptql-bns, seed
+    # conceptql-ysx). config/vocabulary_formats.sha records the commit, and
+    # `just vocabulary-formats <ref>` refreshes both.
+    #
+    # Only the DEFAULT row of a vocabulary is used: the row whose vocabulary_id
+    # is the vocabulary's id. A variant row (ICD9CM.DOTLESS) is never used. The
+    # published patterns carry no anchors, so each one is anchored here as
+    # \A(?:pattern)\z. A vocabulary with no row has no format to check.
+    #
+    # A pattern is case-exact unless its row declares case-insensitivity with
+    # the scoped flag (?i:...) (vocabulary_formats seed vfm-8k1, seed
+    # conceptql-oji ruling "b"). Ruby honors the flag natively, so a pattern
+    # is compiled with no flag of its own: an ignore-case flag would override
+    # what the row says.
+    module Formats
+      DECLARES_CASE_INSENSITIVE = "(?i:"
+
+      class << self
+        # The anchored pattern of a vocabulary as a string, or nil. The
+        # vocabulary_id finds its row in any case.
+        def pattern(vocabulary_id)
+          patterns[vocabulary_id.to_s.downcase]
+        end
+
+        # The anchored pattern as a Regexp, or nil.
+        def regexp(vocabulary_id)
+          (str = pattern(vocabulary_id)) && Regexp.new(str)
+        end
+
+        # True when the vocabulary's row declares case-insensitivity with
+        # (?i:...). The lexicon then matches the vocabulary's codes without
+        # regard to case (LexiconStrategy#code_condition). A vocabulary with no
+        # row is case-exact.
+        def case_insensitive?(vocabulary_id)
+          pattern(vocabulary_id).to_s.include?(DECLARES_CASE_INSENSITIVE)
+        end
+
+        private
+
+        def patterns
+          @patterns ||= CSV.foreach(ConceptQL.vocabulary_formats_file_path, col_sep: "\t", headers: true)
+                           .reject { |row| row["vocabulary_id"].include?(".") }
+                           .to_h { |row| [ row["vocabulary_id"].downcase, "\\A(?:#{row.fetch("regexp")})\\z" ] }
+                           .freeze
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/vocabulary_class_factory.rb

@@ -3,8 +3,7 @@
 require_relative "../behaviors/drugish"
 require_relative "../behaviors/labish"
 require_relative "behaviors/gdmish"
-require_relative "behaviors/omopish"
-require_relative "behaviors/costish"
+require_relative "drg_systems"
 require_relative "../operators/vocabulary"
 
 module ConceptQL
@@ -19,9 +18,8 @@ module ConceptQL
       def get_klasses
         {
           gdm: get_gdm_klass,
-          gdm_wide: get_gdm_klass,
-          omopv4_plus: get_omopv4_plus_klass
-        }.compact
+          gdm_wide: get_gdm_klass
+        }
       end
 
       def get_gdm_klass
@@ -30,16 +28,6 @@ module ConceptQL
         end
       end
 
-      def get_omopv4_plus_klass
-        return nil unless entry.belongs_in_omopv4_plus?
-
-        get_klass do
-          include Behaviors::Omopish
-
-          include Behaviors::Costish if entry.is_costish?
-        end
-      end
-
       def get_klass(&block)
         ventry = entry
         Class.new(ConceptQL::Operators::Vocabulary) do |klass|
@@ -52,7 +40,7 @@ module ConceptQL
           preferred_name entry.preferred_name
           argument :codes, type: :codelist
           aliases entry.aliases
-          predominant_domains entry.predominant_domains
+          predominant_domains(*entry.predominant_domains)
           short_name entry.short_name
           long_name entry.long_name
           conceptql_spec_id "vocabulary"
@@ -75,6 +63,10 @@ module ConceptQL
 
           include ConceptQL::Behaviors::Drugish if entry.is_drugish?
 
+          include DrgSystems::OperatorBehavior if DrgSystems.system(entry.id)
+
+          deprecated DrgSystems::DRG_DEPRECATION if entry.id == DrgSystems::DRG_OPERATOR
+
           def vocab_entry
             self.class.entry
           end

data/lib/conceptql/window/table.rb

@@ -25,8 +25,11 @@ module ConceptQL
 
         rhs = query.db[get_table_window(query)]
 
+        # select_group replaces the select list, so rhs needs no
+        # remove_window_id. Its select_remove asked the database for the
+        # columns of the window table while the SQL was built (bead
+        # conceptql-6k8).
         unless opts[:precalculated_window_id]
-          rhs = remove_window_id(rhs)
           rhs = rhs
                 .select_group(:person_id, :start_date, :end_date)
                 .select_append do
@@ -37,7 +40,7 @@ module ConceptQL
 
         rhs = rhs.as(:r)
 
-        remove_window_id(query)
+        remove_window_id(query, op.dynamic_columns)
           .from_self(alias: :l)
           .join(rhs, expr)
           .select_all(:l)
@@ -52,12 +55,18 @@ module ConceptQL
         db[get_table_window(nil)].select(:person_id)
       end
 
-      def remove_window_id(ds)
+      # columns: the names of the columns of ds, when the caller knows them.
+      # For a ds without a select list, select_remove asks the database for
+      # them, and a table that ds reads may not exist while the SQL is built
+      # (bead conceptql-6k8).
+      def remove_window_id(ds, columns = nil)
         cols = ds.opts[:select]
         if cols && !cols.empty? && cols.any? { |c| !c.is_a?(Symbol) }
           # When select has non-symbol expressions (e.g. CAST...AS),
           # select_remove can't match them by symbol name, so filter manually
           ds.select(*cols.reject { |c| window_id_column?(c) })
+        elsif columns && cols.blank?
+          ds.select(*(columns - [ :window_id ]))
         else
           ds.select_remove(:window_id)
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: conceptql
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Ryan Duryea
@@ -102,7 +102,7 @@ dependencies:
         version: 0.3.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -112,7 +112,7 @@ dependencies:
         version: 0.3.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -328,6 +328,9 @@ files:
 - config/provenance.yml
 - config/race.yml
 - config/vocabularies.csv
+- config/vocabulary_formats.sha
+- config/vocabulary_formats.tsv
+- doc/spec.md
 - lib/conceptql.rb
 - lib/conceptql/behaviors/code_lister.rb
 - lib/conceptql/behaviors/drugish.rb
@@ -344,7 +347,6 @@ files:
 - lib/conceptql/data_model/base.rb
 - lib/conceptql/data_model/gdm.rb
 - lib/conceptql/data_model/gdm_wide.rb
-- lib/conceptql/data_model/omopv4_plus.rb
 - lib/conceptql/database.rb
 - lib/conceptql/date_adjuster.rb
 - lib/conceptql/diagram.rb
@@ -354,13 +356,13 @@ files:
 - lib/conceptql/lexicon/lexicon_ohdsi.rb
 - lib/conceptql/lexicon/lexicon_strategy.rb
 - lib/conceptql/nodifier.rb
+- lib/conceptql/one_query_counts.rb
 - lib/conceptql/operators/after.rb
 - lib/conceptql/operators/any_overlap.rb
 - lib/conceptql/operators/before.rb
 - lib/conceptql/operators/binary_operator_operator.rb
 - lib/conceptql/operators/casting_operator.rb
 - lib/conceptql/operators/co_reported.rb
-- lib/conceptql/operators/concurrent_within.rb
 - lib/conceptql/operators/contains.rb
 - lib/conceptql/operators/count.rb
 - lib/conceptql/operators/date_range.rb
@@ -418,11 +420,6 @@ files:
 - lib/conceptql/query_modifiers/gdm_wide/lab_query_modifier.rb
 - lib/conceptql/query_modifiers/gdm_wide/pos_query_modifier.rb
 - lib/conceptql/query_modifiers/gdm_wide/provenance_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb
 - lib/conceptql/query_modifiers/query_modifier.rb
 - lib/conceptql/rdbms.rb
 - lib/conceptql/rdbms/duckdb.rb
@@ -434,11 +431,11 @@ files:
 - lib/conceptql/utils_cli.rb
 - lib/conceptql/version.rb
 - lib/conceptql/visitors/metadata_extractor.rb
-- lib/conceptql/vocabularies/behaviors/costish.rb
 - lib/conceptql/vocabularies/behaviors/gdmish.rb
-- lib/conceptql/vocabularies/behaviors/omopish.rb
+- lib/conceptql/vocabularies/drg_systems.rb
 - lib/conceptql/vocabularies/dynamic_vocabularies.rb
 - lib/conceptql/vocabularies/entry.rb
+- lib/conceptql/vocabularies/formats.rb
 - lib/conceptql/vocabularies/vocabulary_class_factory.rb
 - lib/conceptql/window.rb
 - lib/conceptql/window/base.rb
@@ -449,8 +446,6 @@ files:
 - schemas/gdm.yml
 - schemas/gdm_wide.yml
 - schemas/ohdsi_vocabs.yml
-- schemas/omopv4_plus.yml
-- schemas/omopv4_plus_more_ignorables.tsv
 homepage: https://github.com/outcomesinsights/conceptql
 licenses:
 - MIT

data/lib/conceptql/data_model/omopv4_plus.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base"
-module ConceptQL
-  module DataModel
-    class Omopv4Plus < Base
-      def base
-        :omop
-      end
-    end
-  end
-end

data/lib/conceptql/operators/concurrent_within.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "operator"
-require_relative "../date_adjuster"
-
-module ConceptQL
-  module Operators
-    class ConcurrentWithin < Operator
-      register __FILE__
-
-      def events_per_patient
-        upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
-      end
-
-      def multiple_vocabularies
-        return true if upstreams.any?(&:multiple_vocabularies)
-
-        vocabularies.length > 1
-      end
-
-      desc "Filters each upstream to only include records where other upstreams' date ranges overlap"
-      option :start, type: :string
-      option :end, type: :string
-      allows_many_upstreams
-      validate_at_least_one_upstream
-      validate_no_arguments
-      validate_option DateAdjuster::VALID_INPUT, :start, :end
-      category "Combine Streams"
-      basic_type :set
-      default_query_columns
-
-      def query(db)
-        datasets = upstreams.map do |stream|
-          stream.evaluate(db)
-        end
-
-        return datasets.first.from_self if datasets.length == 1
-
-        adjusted_start_date = DateAdjuster.new(self, options[:start]).adjust(Sequel[:l][:start_date], true)
-        adjusted_end_date = DateAdjuster.new(self, options[:end]).adjust(Sequel[:l][:end_date])
-
-        datasets = datasets.map do |ds|
-          matching = ds.from_self(alias: :l)
-
-          (datasets - [ ds ]).each do |other|
-            other = other
-                    .from_self(alias: :r)
-                    .where(adjusted_start_date <= Sequel[:r][:start_date])
-                    .where(adjusted_end_date >= Sequel[:r][:end_date])
-                    .select(*matching_columns)
-
-            matching = matching.where(other.where(matching_columns.map do |x|
-              [ Sequel.qualify(:l, x), Sequel.qualify(:r, x) ]
-            end).exists)
-          end
-
-          matching
-        end
-
-        ds, *rest = datasets
-        rest.each do |other|
-          ds = ds.union(other, from_self: nil)
-        end
-
-        ds.from_self
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class AdmissionDateQueryModifier < QueryModifier
-        def self.provided_columns
-          %i[
-            admission_date
-            discharge_date
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          cols.map(&:to_s).any? { |col| col =~ /start_date/i }
-        end
-
-        def modified_query
-          cols = dm.table_cols(source_table).map(&:to_s)
-          return query unless cols.any? { |col| col =~ /start_date/i }
-
-          start_col = cols.find { |col| col =~ /start_date/i }.to_sym
-          end_col = cols.find { |col| col =~ /end_date/i }.to_sym
-
-          query.from_self(alias: :t)
-               .select_all(:t)
-               .select_append(Sequel[:t][start_col].as(:admission_date))
-               .select_append(Sequel.function(:coalesce, Sequel[:t][end_col],
-                                              Sequel[:t][start_col]).as(:discharge_date))
-               .from_self
-        end
-
-        private
-
-        def domain
-          op.domain
-        rescue StandardError
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class DrugQueryModifier < QueryModifier
-        attr_reader :db
-
-        def self.provided_columns
-          %i[
-            drug_name
-            drug_amount
-            drug_amount_units
-            drug_quantity
-            drug_days_supply
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          needed = [ :drug_concept_id ]
-          found = needed & cols
-          needed == found
-        end
-
-        def initialize(*args)
-          super
-          @db = query.db
-        end
-
-        def modified_query
-          return query unless dm.table_cols(source_table).include?(:drug_concept_id)
-
-          query.db
-          drug_concept_ids = query.select(:drug_concept_id).distinct
-          query.from_self(alias: :de)
-               .left_join(micro_table(drug_concept_ids).as(:mt), drug_concept_id: :drug_concept_id)
-               .select_all(:de)
-               .select_append(Sequel[:de][:quantity].as(:drug_quantity))
-               .select_append(Sequel[:de][:days_supply].as(:drug_days_supply))
-               .select_append(Sequel[:mt][:amount_value].as(:drug_amount))
-               .select_append(Sequel[:mt][:amount_unit].as(:drug_amount_units))
-               .select_append(Sequel[:mt][:drug_name].as(:drug_name))
-               .from_self
-        end
-
-        private
-
-        def micro_table(drug_concept_ids)
-          # TODO: Does drug_strength only have RXNORM concept_ids?
-          # TODO: What is vocabulary for units?  Can we shrink concept table to just that vocab before joining?
-          collapsed_strengths = db[:drug_strength]
-                                .where(drug_concept_id: drug_concept_ids)
-                                .select_group(:drug_concept_id)
-                                .select_append(
-                                  Sequel.function(:min, :amount_value).as(:amount_value),
-                                  Sequel.function(:min, :amount_unit).as(:amount_unit),
-                                  Sequel.function(:count, 1).as(:dcount)
-                                )
-
-          value_case = Sequel.case({ 1 => :amount_value }, nil, :dcount).as(:amount_value)
-          unit_case = Sequel.case({ 1 => :amount_unit }, nil, :dcount).as(:amount_unit)
-
-          collapsed_strengths = collapsed_strengths.from_self.select(:drug_concept_id, value_case, unit_case)
-
-          db.from(Sequel[:concept].as(:dc))
-            .left_join(collapsed_strengths, { drug_concept_id: :concept_id }, { table_alias: :ds })
-            .where(Sequel[:dc][:concept_id] => drug_concept_ids)
-            .select(
-              Sequel[:dc][:concept_id].as(:drug_concept_id),
-              Sequel[:dc][:concept_name].as(:drug_name),
-              Sequel[:ds][:amount_value].as(:amount_value),
-              Sequel[:ds][:amount_unit].as(:amount_unit)
-            )
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class PoSQueryModifier < QueryModifier
-        attr_reader :join_id, :table, :source_column, :column
-
-        def self.provided_columns
-          [ :visit_source_concept_id ]
-        end
-
-        def self.has_required_columns?(cols)
-          needed = [ :visit_occurrence_id ]
-          found = needed & cols
-          needed == found
-        end
-
-        def initialize(*args)
-          super
-          @column = :visit_source_concept_id
-          @join_id = :visit_occurrence_id
-          @table = :visit_occurrence
-          @source_column = :visit_source_concept_id
-        end
-
-        def modified_query
-          return query unless dm.table_cols(source_table).include?(join_id)
-          return query if dm.table_cols(source_table).include?(column)
-
-          left_alias = "tab1".to_sym
-          right_alias = "tab2".to_sym
-
-          extra_table = query.db.from(table).select(Sequel.as(source_column, column), join_id)
-          ds = query.from_self(alias: left_alias)
-          ds.left_join(extra_table.as(right_alias),
-                       Sequel.qualify(left_alias, join_id) => Sequel.qualify(right_alias, join_id))
-            .select_all(left_alias)
-            .select_append(Sequel.qualify(right_alias, column).as(column))
-            .from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class ProvenanceQueryModifier < QueryModifier
-        RELATED_COLUMNS = %w[
-          condition_type_concept_id
-          procedure_type_concept_id
-          death_type_concept_id
-          observation_type_concept_id
-          drug_type_concept_id
-        ].sort.map(&:to_sym)
-
-        attr_reader :db
-
-        def self.provided_columns
-          %i[
-            file_provenance_type
-            code_provenance_type
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          !(RELATED_COLUMNS & cols).empty?
-        end
-
-        def initialize(*args)
-          super
-          @db = query.db
-        end
-
-        def modified_query
-          return query unless self.class.has_required_columns?(dm.table_cols(source_table))
-
-          query.select_append(Sequel[dm.file_provenance_type_column(query, source_table)].as(:file_provenance_type),
-                              Sequel[dm.code_provenance_type_column(query,
-                                                                    source_table)].as(:code_provenance_type)).from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class ProviderQueryModifier < QueryModifier
-        def self.provided_columns
-          [ :provider_id ]
-        end
-
-        def self.has_required_columns?(cols)
-          cols.find { |col| col.to_s =~ /provider_id/ }
-        end
-
-        def modified_query
-          col = self.class.has_required_columns?(dm.table_cols(source_table))
-          return query if col.nil? || col.to_sym == :provider_id
-
-          query.select_append(Sequel[col].as(:provider_id)).from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/vocabularies/behaviors/costish.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../../operators/vocabulary"
-
-module ConceptQL
-  module Vocabularies
-    module Behaviors
-      module Costish
-        def query(db)
-          costs = db[:procedure_cost].where(column_to_search => values).select(:procedure_occurrence_id)
-          db[:procedure_occurrence].where(procedure_occurrence_id: costs)
-        end
-
-        def table
-          :procedure_occurrence
-        end
-
-        def column_to_search
-          vocab_entry.id == "revenue_code" ? :revenue_code_source_value : :disease_class_source_value
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/vocabularies/behaviors/omopish.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module ConceptQL
-  module Vocabularies
-    module Behaviors
-      module Omopish
-        # Overrides Operator#tables
-        def tables
-          domains
-        end
-
-        def vocabulary_condition
-          { dm.source_vocabulary_id(domain) => vocabulary_id.to_i }
-        end
-
-        def where_clause(_db)
-          conds = vocabulary_condition
-          conds[dm.source_value_column(domain)] = arguments unless select_all?
-          conds
-        end
-
-        def query_cols
-          dm.table_columns(domain)
-        end
-
-        def translated_vocabulary_id
-          vocab_entry.omopv4_id
-        end
-      end
-    end
-  end
-end