conceptql 0.3.0 → 0.4.0

data/config/race.yml

@@ -0,0 +1,50 @@
+8657:
+  - 38003572
+  - 38003573
+8515:
+  - 38003574
+  - 38003575
+  - 38003576
+  - 38003577
+  - 38003578
+  - 38003579
+  - 38003580
+  - 38003581
+  - 38003582
+  - 38003583
+  - 38003584
+  - 38003585
+  - 38003586
+  - 38003587
+  - 38003588
+  - 38003589
+  - 38003590
+  - 38003591
+  - 38003592
+  - 38003593
+  - 38003594
+  - 38003595
+  - 38003596
+  - 38003597
+8516:
+  - 38003598
+  - 38003599
+  - 38003600
+  - 38003601
+  - 38003602
+  - 38003603
+  - 38003604
+  - 38003605
+  - 38003606
+  - 38003607
+  - 38003608
+  - 38003609
+8557:
+  - 38003610
+  - 38003611
+  - 38003612
+  - 38003613
+8527:
+  - 38003614
+  - 38003615
+  - 38003616

data/config/vocabularies.csv

@@ -0,0 +1,87 @@
+id,omopv4_vocabulary_id,vocabulary_full_name,vocabulary_short_name,domain,hidden,format_regexp,aliases,requires_lexicon
+ADMSRCE,1000000,ADMSRCE,,,,,,
+ADMTYPE,1000001,ADMTYPE,,,,,,
+ATC,21,WHO Anatomic Therapeutic Chemical Classification,WHO ATC,drug_exposure,,"^[ABCDGHJLMNPRSVabcdghjlmnprsv](\d{2}([A-Za-z]{1,2}(\d{2})?)?)?$",,
+BETOS,1000002,BETOS,,,,,,
+Clinical Code Type,1000004,Clinical Code Type,,,,,,
+Concept Class,68,OMOP Concept Class,,,Y,,,
+Condition Type,37,OMOP Condition Occurrence Type,,condition_occurrence,Y,,,
+CPT4,4,Current Procedural Terminology version 4 (AMA),CPT,procedure_occurrence,,"^\w\d{3}\w$",cpt,
+CPT4_HCPCS,,CPT4 and HCPCS merged (OI supplemental),CPT4/HCPCS,procedure_occurrence,,"^\w\d{3}\w$",,Y
+CS_Schema,1000003,CS_Schema,,,,,,
+Currency,65,International Currency Symbol (ISO 4217),,,Y,,,
+Death Type,45,OMOP Death Type,,death,Y,,,
+Device Type,63,OMOP Device Type,,observation,Y,,,
+Domain,59,OMOP Domain,,,Y,,,
+DRG,40,Diagnosis-related group (CMS),DRG,procedure_occurrence,,"^\d{3}$",,
+DrugTextString,26400,Drug Text String,Drug Text String,drug_exposure,,,,
+Drug Type,36,OMOP Drug Exposure Type,,drug_exposure,Y,,,
+DURATION_UNIT,1000005,DURATION_UNIT,,,,,,
+Ethnicity,44,OMOP Ethnicity,,person,Y,,,
+Gemscript,56,,Gemscript,drug_exposure,,"^\d{8}$",,
+Gender,12,OMOP Gender,,person,Y,,,
+HCFASPEC,1000006,HCFASPEC,,,,,,
+HCFATYPE,1000007,HCFATYPE,,,,,,
+HCPCS,5,Healthcare Common Procedure Coding System (CMS),HCPCS,procedure_occurrence,,"^[A-Za-z]\d{4}$",,
+ICD03_Morphology,1000008,ICD03_Morphology,,,,"^\d{4}/\d$",,
+ICD03_Topography,1000009,ICD03_Topography,,,,"^[A-Za-z]\d{2}(\.\d{1,2})?$",,
+ICD10,34,"International Classification of Diseases, Tenth Revision (WHO)",ICD-10 WHO,condition_occurrence,,"^([A-Za-z][0-9][0-9A-Za-z](\.\w{1,4})?|[A-Za-z][0-9][0-9A-Za-z]-[A-Za-z][0-9][0-9A-Za-z]|(?:I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX|XXI|XXII))$",,
+ICD10PCS,35,"ICD-10 Procedure Coding System (CMS)",ICD-10 PCS,procedure_occurrence,,"^[0-9BCDFGHX][0-9A-HJ-NP-Z]{6}$",icd10_pcs,
+ICD10CM,70,"International Classification of Diseases, Tenth Revision, Clinical Modification (NCHS)",ICD-10 CM,condition_occurrence,,"^(?:QA[0-9]|[A-Za-z][0-9][0-9ABab])(\.[0-9A-Za-z]{1,4})?$",,
+ICD9CM,2,"International Classification of Diseases, Ninth Revision, Clinical Modification, Volume 1 and 2 (NCHS)",ICD-9 CM,condition_occurrence,,"^(V\d{2}(\.\d{1,2})?|\d{3}(\.\d{1,2})?|E\d{3}(\.\d)?)$",icd9,
+ICD9Proc,3,"International Classification of Diseases, Ninth Revision, Clinical Modification, Volume 3 (NCHS)",ICD-9 Proc,procedure_occurrence,,"^\d{2}(\.\d{1,2})?$",icd9_procedure,
+LOINC,6,Logical Observation Identifiers Names and Codes (Regenstrief Institute),,observation,,"^\w{3,7}-\w$",,
+GDM_CEN_TRACT_MEAS,0,Census Tract Measures from SEER,,observation,,,,
+SEER_YOST_ACS_06_10,0,SEER's Yost Index calculated from ACS 2006-2010,,observation,,,,
+SEER_YOST_ACS_10_14,0,SEER's Yost Index calculated from ACS 2010-2014,,observation,,,,
+SEER_YOST_ACS_13_17,0,SEER's Yost Index calculated from ACS 2013-2017,,observation,,,,
+MARCLA_OBS_TYPE,0,Market Clarity Observation Type,,observation,,,,
+Meas Type,64,OMOP Measurement Type,,observation,Y,,,
+NDC,9,National Drug Code (FDA and manufacturers),,drug_exposure,,"^(\d{5}\w{4}|\d{5}\w{6})$",,
+Note Type,58,OMOP Note Type,,,Y,,,
+NUCC,47,National Uniform Claim Committee Health Care Provider Taxonomy Code Set (NUCC),,,Y,,,
+Observation Type,39,OMOP Observation Type,,observation,Y,,,
+Obs Period Type,61,OMOP Observation Period Type,,observation_period,Y,,,
+OI_FILE_TYPE,1000010,OI_FILE_TYPE,,,,,,
+OI_INFORMATION_TYPE,1000011,OI_INFORMATION_TYPE,,,,,,
+OI_PROV,1000012,OI_PROV,,,,,,
+Place of Service,14,Place of Service Codes for Professional Claims (CMS),,visit_occurrence,Y,,,
+PLCSRVC,1000013,PLCSRVC,,,,,,
+Procedure Type,38,OMOP Procedure Occurrence Type,,procedure_occurrence,Y,,,
+PROVIDER_FACILITY_TYPE,1000014,PROVIDER_FACILITY_TYPE,,,,,,
+Race,13,Race and Ethnicity Code Set (USBC),,person,Y,,,
+Relationship,66,OMOP Relationship,,,Y,,,
+Revenue Code,43,UB04/CMS1450 Revenue Codes (CMS),Revenue Code,procedure_occurrence,,,revenue_code,
+RxNorm,8,RxNorm (NLM),,drug_exposure,,"^\d{2,8}$",,
+SEER_ADJAJC6M,1000015,SEER_ADJAJC6M,,,,,,
+SEER_ADJAJC6N,1000016,SEER_ADJAJC6N,,,,,,
+SEER_ADJAJC6T,1000017,SEER_ADJAJC6T,,,,,,
+SEER_BEH,1000018,SEER_BEH,,,,,,
+SEER_CENSUS_POV_IND,1000019,SEER_CENSUS_POV_IND,,,,,,
+SEER_DAJCC7,1000020,SEER_DAJCC7,,,,,,
+SEER_DAJCCSTG,1000021,SEER_DAJCCSTG,,,,,,
+SEER_DXCONF,1000022,SEER_DXCONF,,,,,,
+SEER_GRADE,1000023,SEER_GRADE,,,,,,
+SEER_LAT,1000024,SEER_LAT,,,,,,
+SEER_MARST,1000025,SEER_MARST,,,,,,
+SEER_MSEX,1000026,SEER_MSEX,,,,,,
+SEER_NHIADE,1000027,SEER_NHIADE,,,,,,
+SEER_NOSRG,1000028,SEER_NOSRG,,,,,,
+SEER_RAD,1000029,SEER_RAD,,,,,,
+SEER_RADSURG,1000030,SEER_RADSURG,,,,,,
+SEER_SEQ,1000031,SEER_SEQ,,,,,,
+SEER_SITERWHO,1000032,SEER_SITERWHO,,,,,,
+SEER_SRACE,1000033,SEER_SRACE,,,,,,
+SEER_SRC,1000034,SEER_SRC,,,,,,
+SEER_SXPRIF,1000035,SEER_SXPRIF,,,,,,
+SEER_SXSITF,1000036,SEER_SXSITF,,,,,,
+SEER_TYPEFU,1000037,SEER_TYPEFU,,,,,,
+SEER_URBRUR,1000038,SEER_URBRUR,,,,,,
+SNOMED,1,SNOMED-CT,SNOMED-CT,condition_occurrence,,"^\d{6,18}$",,
+Specialty,48,Medicare provider/supplier specialty codes (CMS),,provider,Y,,,
+STUS_CD,1000039,STUS_CD,,,,,,
+TYPESRVC,1000040,TYPESRVC,,,,,,
+UCUM,11,Unified Code for Units of Measure (Regenstrief Institute),,,Y,,,
+Visit,24,OMOP Visit,,visit_occurrence,Y,,,
+Visit Type,62,OMOP Visit Type,,visit_occurrence,Y,,,
+Vocabulary,67,OMOP Vocabulary,,,Y,,,

data/lib/conceptql/behaviors/code_lister.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    module CodeLister
+      ConceptCode = Struct.new(:vocabulary, :code, :description) do
+        def to_s
+          if description
+            "#{vocabulary} #{code}: #{description}"
+          else
+            "#{vocabulary} #{code}"
+          end
+        end
+      end
+
+      def code_list(db)
+        describe_codes(db, arguments).map do |code, desc|
+          ConceptCode.new(preferred_name, code, desc)
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/drugish.rb

@@ -1,12 +1,17 @@
+# frozen_string_literal: true
+
 module ConceptQL
-  module Drugish
-    def self.included(base)
-      base.require_column(:drug_name)
-      base.require_column(:drug_amount)
-      base.require_column(:drug_amount_units)
-      base.require_column(:quantity)
-      base.require_column(:days_supply)
-      base.query_columns(:drug_exposure, :drug_appendix)
+  module Behaviors
+    module Drugish
+      def self.included(base)
+        base.define_method(:column_family) { "drug" }
+        base.require_column(:drug_name)
+        base.require_column(:drug_amount)
+        base.require_column(:drug_amount_units)
+        base.require_column(:drug_quantity)
+        base.require_column(:drug_strength_source_value)
+        base.require_column(:drug_days_supply)
+      end
     end
   end
 end

data/lib/conceptql/behaviors/labish.rb

@@ -1,13 +1,19 @@
+# frozen_string_literal: true
+
 module ConceptQL
-  module Labish
-    def self.included(base)
-      base.require_column :value_as_number
-      base.require_column :value_as_string
-      base.require_column :value_as_concept_id
-      base.require_column :unit_source_value
-      base.require_column :range_low
-      base.require_column :range_high
+  module Behaviors
+    module Labish
+      def self.included(base)
+        base.define_method(:column_family) { "lab" }
+        base.require_column :value_as_number
+        base.require_column :value_as_string
+        base.require_column :value_as_concept_id
+        base.require_column :value_as_concept
+        base.require_column :unit_concept_id
+        base.require_column :unit_concept
+        base.require_column :range_low
+        base.require_column :range_high
+      end
     end
   end
 end
-

data/lib/conceptql/behaviors/metadatable.rb

@@ -1,27 +1,52 @@
-require 'facets/kernel/meta_def'
-require 'facets/string/snakecase'
-require 'facets/string/titlecase'
+# frozen_string_literal: true
 
 module ConceptQL
   module Metadatable
     def preferred_name(value = nil)
       return @preferred_name unless value
+
       @preferred_name = value
     end
 
+    def aliases(value = nil)
+      return @aliases unless value
+
+      @aliases = value
+    end
+
     def desc(value = nil)
       return @desc unless value
+
       @desc = value
     end
 
+    def short_name(value = nil)
+      return @short_name unless value
+
+      @short_name = value
+    end
+
+    def long_name(value = nil)
+      return @long_name unless value
+
+      @long_name = value
+    end
+
+    def conceptql_spec_id(value = nil)
+      return @conceptql_spec_id unless value
+
+      @conceptql_spec_id = value
+    end
+
     def predominant_domains(*values)
       return @predominant_domains if values.empty?
+
       @predominant_domains = values
     end
 
     def argument(name, options = {})
       (@arguments ||= [])
-      @arguments << [name, auto_label(name, options)]
+      @arguments << [ name, auto_label(name, options) ]
     end
 
     def option(name, options = {})
@@ -29,26 +54,33 @@ module ConceptQL
       @options[name] = auto_label(name, options)
     end
 
+    # Takes an option that a subclass inherited out of its metadata
+    def remove_option(name)
+      @options.delete(name)
+    end
+
     def auto_label(name, opts = {})
       return opts if opts[:label]
-      return opts.merge(label: name.to_s.split('_').join(' ').titlecase) unless opts[:type] == :codelist
-      opts.merge(label: pref_name + " Codes")
+      return opts.merge(label: name.to_s.split("_").map(&:capitalize).join(" ")) unless opts[:type] == :codelist
+
+      opts.merge(label: "#{pref_name} Codes")
     end
 
     def domains(*domain_list)
       @domains = domain_list
-      define_method(:domains) do |db|
+      define_method(:domains) do |_db|
         domain_list
       end
-      if domain_list.length == 1
-        define_method(:domain) do
-          domain_list.first
-        end
+      return unless domain_list.length == 1
+
+      define_method(:domain) do
+        domain_list.first
       end
     end
 
     def basic_type(value = nil)
       return @basic_type unless value
+
       @basic_type = value
     end
 
@@ -61,7 +93,7 @@ module ConceptQL
     end
 
     def just_class_name
-      self.to_s.split('::').last
+      to_s.split("::").last
     end
 
     def humanized_class_name
@@ -78,6 +110,7 @@ module ConceptQL
     end
 
     def inherited(upstream)
+      super
       (@options || {}).each do |name, opt|
         upstream.option name, opt
       end
@@ -100,26 +133,60 @@ module ConceptQL
       @preferred_name || humanized_class_name
     end
 
+    def spec_id
+      "#{@conceptql_spec_id || pref_name} operator".gsub(/\s+/, "-").downcase
+    end
+
     def to_metadata(name, opts = {})
       derive_metadata_from_validations
       warn_about_missing_metadata if opts[:warn]
       {
         name: name,
         preferred_name: pref_name,
-        operation: just_class_name.snakecase,
+        short_name: short_name,
+        long_name: long_name,
+        operation: just_class_name.underscore,
         min_upstreams: @max_upstreams || 0,
         max_upstreams: @max_upstreams || 0,
         arguments: @arguments || [],
-        options: @options || {},
+        options: process_options(@options),
         predominant_domains: @domains || @predominant_domains || [],
         desc: get_desc,
         categories: @categories || [],
-        basic_type: @basic_type
+        basic_type: @basic_type,
+        deprecated: nil, # kept for metadata consumers; no operator is deprecated
+        arity: arity,
+        aliases: @aliases,
+        spec_id: spec_id
       }
     end
 
+    def arity
+      case basic_type
+      when :set
+        -1
+      when :temporal, :cast
+        1
+      when :selection, nil
+        0
+      when :filter
+        2
+      else
+        raise "#{name} has unknown basic_type: #{basic_type.pretty_inspect}"
+      end
+    end
+
     def get_desc
-      @desc ||= standard_description
+      @get_desc ||= standard_description
+    end
+
+    def process_options(options)
+      return {} if options.nil? || options.empty?
+
+      # Build new hashes: the option hashes can be shared with a parent class
+      options.transform_values do |v|
+        v[:options].respond_to?(:call) ? v.merge(options: v[:options].call) : v
+      end
     end
 
     def warn_about_missing_metadata
@@ -127,56 +194,57 @@ module ConceptQL
       missing << :categories if (@categories || []).empty?
       missing << :desc if get_desc.empty?
       missing << :basic_type unless @basic_type
-      puts "#{just_class_name} is missing #{missing.join(", ")}" unless missing.empty?
+      puts "#{just_class_name} is missing #{missing.join(', ')}" unless missing.empty?
     end
 
+    # Each validation is [meth, *args]. Every argument reaches the hook, so
+    # validate_required_options :left, :right marks both options.
     def derive_metadata_from_validations
-      instance_variable_get(:@validations).each do |meth, args|
-        meth = meth.to_s + "_to_metadata"
-        send(meth, args) if respond_to?(meth)
+      @validations.each do |meth, *args|
+        hook = "#{meth}_to_metadata"
+        send(hook, *args) if respond_to?(hook)
       end
     end
 
-    def validate_no_upstreams_to_metadata(*args)
+    def validate_no_upstreams_to_metadata(*_args)
       @min_upstreams = 0
       @max_upstreams = 0
     end
 
-    def validate_one_upstream_to_metadata(*args)
+    def validate_one_upstream_to_metadata(*_args)
       @min_upstreams = 1
       @max_upstreams = 1
     end
 
-    def validate_at_least_one_upstream_to_metadata(*args)
+    def validate_at_least_one_upstream_to_metadata(*_args)
       @min_upstreams = 1
       @max_upstreams = 99
     end
 
-    def validate_at_most_one_upstream_to_metadata(*args)
+    def validate_at_most_one_upstream_to_metadata(*_args)
       @min_upstreams = 0
       @max_upstreams = 1
     end
 
-    def validate_no_arguments_to_metadata(*args)
+    def validate_no_arguments_to_metadata(*_args)
       @arguments = []
     end
 
-    def validate_required_options_to_metadata(*args)
-      args.each do |opt_name|
-        @options[opt_name][:required] = true
+    # A subclass shares the option hashes that it inherits (see #inherited),
+    # so mark a copy. Writing into the shared hash would mark the option as
+    # required for the parent class too.
+    def validate_required_options_to_metadata(*names)
+      names.each do |name|
+        @options[name] = @options.fetch(name).merge(required: true)
       end
     end
 
     def standard_description
-      table = (!@domains.nil? && @domains.first)
-      table ||= (!predominant_domains.nil? && predominant_domains.first)
-      raise "Can't create description for #{pref_name}" unless table
-
-      "Selects results from the #{table} table where #{table}'s source value matches the given #{pref_name} codes."
+      "Selects records based on the #{pref_name} vocabulary."
     end
 
     def no_desc
-      desc ''
+      desc ""
     end
   end
 end

data/lib/conceptql/behaviors/provenanceable.rb

@@ -1,30 +1,26 @@
+# frozen_string_literal: true
+
 module ConceptQL
   module Provenanceable
-    def to_concept_id(ctype)
-      ctype = ctype.to_s.downcase
-      return ctype.to_i unless ctype.to_i.zero?
-      position = nil
-      if ctype =~ /(\d|_primary)$/ && ctype.count('_') > 1
-        parts = ctype.split('_')
-        position = parts.pop.to_i
-        position -= 1 if needs_adjustment?(ctype)
-        ctype = parts.join('_')
-      end
-      retval = concept_ids[ctype.to_sym]
-      return retval[position] if position
-      return retval
+    def self.included(base)
+      base.require_column :file_provenance_type
+      base.require_column :code_provenance_type
     end
 
-    def concept_ids
-      @concept_ids ||= Psych.load_file(config_dir + 'provenance.yml')
+    def prov_of(db, ancestors)
+      dm.descendants_of(db,
+                        dm.concept_ids(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten,
+                                       ancestors))
     end
 
-    def config_dir
-      Pathname.new(__FILE__).dirname + '..' + '..' + '..' + 'config'
+    def build_where_from_codes(db, codes)
+      Sequel.|(
+        { file_provenance_type: prov_of(db, codes) }, { code_provenance_type: prov_of(db, codes) }
+      )
     end
 
-    def needs_adjustment?(ctype)
-      ctype =~ /^outpatient/ || ctype =~ /^carrier/
+    def find_bad_keywords(db, codes)
+      codes - dm.concept_codes(db, [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten)
     end
   end
 end

data/lib/conceptql/behaviors/single_event.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    module SingleEvent
+      def events_per_patient
+        :single
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/timeless.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    module Timeless
+      def timeless?
+        true
+      end
+    end
+  end
+end