conceptql 0.6.0 → 0.7.0

data/lib/conceptql/operators/vocabulary.rb

@@ -5,16 +5,54 @@ require_relative "operator"
 module ConceptQL
   module Operators
     class Vocabulary < Operator
+      # The vocabulary operators share one reference section (spec_id
+      # vocabulary-operator): each generated vocabulary operator, read and
+      # the multiple vocabulary operators. So the section's texts are written
+      # on this class: `returns`, the meaning: of the codes argument, and each
+      # `example`. An operator of the section extends this module and serves
+      # the returns text, and only the examples whose statement starts with
+      # its own name, so no operator shows an example of another one.
+      module SharedDocumentation
+        def get_returns
+          super || Vocabulary.returns
+        end
+
+        def examples_for(name)
+          Vocabulary.examples.select { |example| example[:statement].first.to_s == name }
+        end
+      end
+
+      extend SharedDocumentation
+
       include ConceptQL::Behaviors::Windowable
       include ConceptQL::Behaviors::CodeLister
 
       category "Select by Clinical Codes"
       basic_type :selection
-      desc "Selects records based on the given vocabulary."
+      desc "Selects the records whose code is one of the given codes in a vocabulary, such as ICD-9-CM diagnoses or CPT procedures. " \
+           "Each vocabulary has its own operator, and a few operators search several vocabularies at once."
       validate_no_upstreams
       validate_at_least_one_argument
       validate_codes_match
 
+      # VocabularyClassFactory gives each vocabulary operator a copy of this
+      # argument, with the label "<preferred name> Codes"
+      argument :codes, type: :codelist,
+                       meaning: "One or more codes of the vocabulary. Write each code as the vocabulary writes it, " \
+                                "including any dots: 250.01, not 25001. A code matches exactly, including case, " \
+                                "and it does not select its child codes. A code alone of * selects every record " \
+                                "of the vocabulary, and there is no other wildcard. A code that does not have the " \
+                                "vocabulary's format matches nothing and gives a warning."
+
+      returns "Every record whose code is one of the given codes in the operator's vocabulary. " \
+              "The type of the records depends on the vocabulary, for example condition_occurrence for icd9cm " \
+              "and procedure_occurrence for cpt4. An operator does not follow mappings to other vocabularies."
+      example "All records with the ICD-9-CM diagnosis code 412 (old myocardial infarction)", [ "icd9cm", "412" ]
+      example "All MS-DRG 205 records: other respiratory system diagnoses with a major complication or comorbidity",
+              [ "ms_drg", "205" ]
+      # The test data has no DRG record before 2007-10-01
+      example "All CMS-DRG 205 records (a liver disorder)", [ "cms_drg", "205" ], help: false
+
       def source_table
         domain_map(op_name)
       end
@@ -31,25 +69,61 @@ module ConceptQL
         vocab_entry.predominant_domains.map(&:to_sym)
       end
 
+      # Production ETL data stores CPT-4 and HCPCS codes only under the merged
+      # vocabulary CPT4_HCPCS. cpt and hcpcs read only their own vocabulary,
+      # so they select no rows there (known issue conceptql-rpy3). Each key is
+      # a vocabulary whose codes can be stored under the merged vocabulary,
+      # and each value names that vocabulary and the operator that reads it.
+      MERGED_VOCABULARIES = {
+        "CPT4" => { vocabulary_id: "CPT4_HCPCS", operator: "cpt_or_hcpcs" },
+        "HCPCS" => { vocabulary_id: "CPT4_HCPCS", operator: "cpt_or_hcpcs" }
+      }.freeze
+      NO_RECORDS_IN_DATA = "the data has no records of this vocabulary"
+
+      # MultipleVocabularies passes skip_merged_vocabulary_check to its
+      # vocabulary operators. It reads the merged vocabulary itself, and it
+      # would drop this warning anyway (see #warn_about_merged_vocabulary).
       def additional_validation(db, opts = {})
-        return unless add_warnings?(db, opts) && !select_all?
+        return unless add_warnings?(db, opts)
+
+        warn_about_merged_vocabulary(db) unless opts[:skip_merged_vocabulary_check]
+        return if select_all?
 
         args = arguments.dup
         args -= bad_arguments
 
-        missing_args = args - dm.known_codes(db, vocabulary_id, args)
+        # A statement warns about a code in another case than the stored one,
+        # so its lookups do not raise CodeCaseMismatchError (bead conceptql-rz8s).
+        missing_args = args - dm.known_codes(db, vocabulary_id, args, check_case: false)
         add_warning(UNKNOWN_CODE, *missing_args) unless missing_args.empty?
 
         warn_about_reused_codes(db, args - missing_args)
       end
 
+      # Warns when the data has no record of this vocabulary but has records
+      # of the merged vocabulary that holds its codes, and the Database's
+      # lexicon carries that merged vocabulary (bead conceptql-k5ho). The
+      # warning names the operator to use instead. The test fixture keeps
+      # CPT4 and HCPCS rows, so it never shows this warning.
+      def warn_about_merged_vocabulary(db)
+        merged = MERGED_VOCABULARIES[vocabulary_id]
+        return unless merged && nodifier.operators.key?(merged[:vocabulary_id].downcase)
+
+        table = db[criterion_table]
+        return unless table.where(vocabulary_condition).empty?
+        return if table.where(vocabulary_condition.transform_values { merged[:vocabulary_id] }).empty?
+
+        add_warning(NO_RECORDS_IN_DATA,
+                    "the data stores #{vocabulary_id} codes as #{merged[:vocabulary_id]}, so use #{merged[:operator]}")
+      end
+
       # A code that the vocabulary reused for a new meaning selects records of
       # both meanings: DRG 410 selects chemotherapy stays before 2007-10-01 and
       # biliary tract procedures after it (conceptql-ttk). Each item names the
       # code and all of its names, so the user sees the problem without a
       # vocabulary lookup.
       def warn_about_reused_codes(db, codes)
-        meanings = dm.code_meanings(db, vocabulary_id, codes)
+        meanings = dm.code_meanings(db, vocabulary_id, codes, check_case: false)
         reused = codes.filter_map do |code|
           names = meanings[code.downcase]
           "#{code}: #{names.join(' | ')}" if names
@@ -60,7 +134,7 @@ module ConceptQL
       def describe_codes(db, codes)
         return [ [ "*", "ALL CODES" ] ] if select_all?
 
-        found_codes = dm.concepts_to_codes(db, vocabulary_id, codes)
+        found_codes = dm.concepts_to_codes(db, vocabulary_id, codes, check_case: false)
 
         found_codes + (codes - found_codes.map(&:first)).zip([])
       end
@@ -112,7 +186,7 @@ module ConceptQL
         wheres = vocabulary_condition
 
         unless select_all?
-          concept_ids = dm.concept_ids(db, vocabulary_id, arguments)
+          concept_ids = dm.concept_ids(db, vocabulary_id, arguments, check_case: false)
 
           wheres = wheres.merge({ clinical_code_concept_id: concept_ids })
         end

data/lib/conceptql/paths.rb

@@ -51,7 +51,15 @@ module ConceptQL
     # The ConceptQL specification, shipped in the gem for applications that
     # serve it to readers (the Jigsaw diagram editor serves it over MCP).
     def spec_path
-      root / "doc" / "spec.md"
+      root / "docs" / "spec" / "README.md"
+    end
+
+    # The directory of the spec's example files, one <spec_id>.json for each
+    # published spec_id: the title, statement, help flag, result columns and
+    # rows, and diagram tree (conceptql-diagram/v1) of each example. An
+    # application reads the file of the operator it shows.
+    def spec_examples_path
+      root / "docs" / "spec" / "examples"
     end
   end
 end

data/lib/conceptql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConceptQL
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/conceptql/vocabularies/drg_systems.rb

@@ -76,7 +76,7 @@ module ConceptQL
                                       omopv5_vocabulary_id: VOCABULARY_ID,
                                       preferred_name: system.name,
                                       vocabulary_short_name: system.name,
-                                      vocabulary_long_name: system.long_name))
+                                      vocabulary_full_name: system.long_name))
           end
         end
       end
@@ -95,7 +95,8 @@ module ConceptQL
         def describe_codes(db, codes)
           return super if select_all?
 
-          in_system = dm.concepts_to_codes_in_class(db, vocabulary_id, drg_system.concept_class_id, codes)
+          in_system = dm.concepts_to_codes_in_class(db, vocabulary_id, drg_system.concept_class_id, codes,
+                                                    check_case: false)
           return super if in_system.blank?
 
           named = in_system.map { |code, _| code.downcase }

data/lib/conceptql/vocabularies/entry.rb

@@ -13,7 +13,7 @@ module ConceptQL
         omopv5_vocabulary_id
         domain
         vocabulary_short_name
-        vocabulary_long_name
+        vocabulary_full_name
       ].freeze
 
       METHODS.each do |meth|
@@ -28,8 +28,14 @@ module ConceptQL
         predominant_domains
       ]
 
+      # A blank domain is no domain (bead conceptql-7azx). The GDM vocabulary
+      # table gives a vocabulary without a domain the domain '', and that
+      # became the criterion_domain of its operators. Without the key, the
+      # vocabulary selects condition_occurrence records (predominant_domains),
+      # and a merge keeps the other entry's domain.
       def initialize(hash)
         @hash = hash
+        @hash.delete(:domain) if @hash[:domain].blank?
         @hash[:omopv5_vocabulary_id] ||= @hash[:id]
         @hash[:aliases_arr] ||= []
         @hash[:aliases_arr] |= (@hash[:aliases] || "").split(";")
@@ -63,12 +69,17 @@ module ConceptQL
         Formats.pattern(omopv5_id)
       end
 
+      # A vocabulary without a short name has none. It does not fall back to
+      # the long name: the operator's preferred_name falls back to the
+      # vocabulary ID (Operators::Vocabulary#preferred_name).
       def short_name
-        vocabulary_short_name || vocabulary_long_name
+        vocabulary_short_name
       end
 
+      # The column vocabulary_full_name of config/vocabularies.csv and of both
+      # lexicons (bead conceptql-txmo)
       def long_name
-        vocabulary_long_name || vocabulary_short_name
+        vocabulary_full_name || vocabulary_short_name
       end
 
       # A preferred_name in the hash wins: ms_drg and cms_drg select vocabulary
@@ -102,10 +113,6 @@ module ConceptQL
         hash[:domain].present?
       end
 
-      def hidden?
-        hash[:hidden].present?
-      end
-
       def is_labish?
         hash[:domain] =~ /observation|measurement/i
       end
@@ -114,10 +121,6 @@ module ConceptQL
         hash[:domain] =~ /drug_exposure/i
       end
 
-      def visible?
-        !hidden?
-      end
-
       def omopv5_id
         omopv5_vocabulary_id || id
       end

data/lib/conceptql/vocabularies/vocabulary_class_factory.rb

@@ -38,7 +38,8 @@ module ConceptQL
           end
 
           preferred_name entry.preferred_name
-          argument :codes, type: :codelist
+          # The label is "<preferred name> Codes" (Metadatable#auto_label)
+          argument :codes, ConceptQL::Operators::Vocabulary.argument_options(:codes).except(:label)
           aliases entry.aliases
           predominant_domains(*entry.predominant_domains)
           short_name entry.short_name

data/lib/conceptql.rb

@@ -30,6 +30,9 @@ end
 module ConceptQL
   DEFAULT_DATA_MODEL = :gdm
 
+  # The diagram renderer (docs/diagram_renderer.md) loads on first use.
+  autoload :DiagramRenderer, "conceptql/diagram_renderer"
+
   def self.avoid_ctes?
     ENV["CONCEPTQL_AVOID_CTES"] == "true"
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: conceptql
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Ryan Duryea
@@ -183,6 +183,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -322,6 +336,14 @@ files:
 - CHANGELOG.md
 - LICENSE
 - README.md
+- assets/diagram/diagram.css
+- assets/diagram/fonts/OFL.txt
+- assets/diagram/fonts/Roboto-400.ttf
+- assets/diagram/fonts/Roboto-700.ttf
+- assets/diagram/fonts/Roboto-Italic-400.ttf
+- assets/diagram/fonts/Roboto-Italic-Latin.woff2
+- assets/diagram/fonts/Roboto-Normal-Latin.woff2
+- assets/diagram/operators.svg
 - bin/conceptql
 - bin/conceptql_utils
 - config/multiple_vocabularies.csv
@@ -330,7 +352,43 @@ files:
 - config/vocabularies.csv
 - config/vocabulary_formats.sha
 - config/vocabulary_formats.tsv
-- doc/spec.md
+- docs/spec/README.md
+- docs/spec/examples/after-operator.json
+- docs/spec/examples/any-overlap-operator.json
+- docs/spec/examples/before-operator.json
+- docs/spec/examples/co-reported-operator.json
+- docs/spec/examples/contains-operator.json
+- docs/spec/examples/date-range-operator.json
+- docs/spec/examples/death-operator.json
+- docs/spec/examples/during-operator.json
+- docs/spec/examples/episode-operator.json
+- docs/spec/examples/equal-operator.json
+- docs/spec/examples/ethnicity-operator.json
+- docs/spec/examples/except-operator.json
+- docs/spec/examples/filter-operator.json
+- docs/spec/examples/first-operator.json
+- docs/spec/examples/gender-operator.json
+- docs/spec/examples/hospice-operator.json
+- docs/spec/examples/hospitalization-operator.json
+- docs/spec/examples/information-periods-operator.json
+- docs/spec/examples/last-operator.json
+- docs/spec/examples/match-operator.json
+- docs/spec/examples/nth-occurrence-operator.json
+- docs/spec/examples/numeric-filter-operator.json
+- docs/spec/examples/one-in-two-out-operator.json
+- docs/spec/examples/person-filter-operator.json
+- docs/spec/examples/person-operator.json
+- docs/spec/examples/place-of-service-filter-operator.json
+- docs/spec/examples/provenance-operator.json
+- docs/spec/examples/provider-filter-operator.json
+- docs/spec/examples/race-operator.json
+- docs/spec/examples/recall-operator.json
+- docs/spec/examples/snf-operator.json
+- docs/spec/examples/time-window-operator.json
+- docs/spec/examples/trim-date-end-operator.json
+- docs/spec/examples/trim-date-start-operator.json
+- docs/spec/examples/union-operator.json
+- docs/spec/examples/vocabulary-operator.json
 - lib/conceptql.rb
 - lib/conceptql/behaviors/code_lister.rb
 - lib/conceptql/behaviors/drugish.rb
@@ -342,6 +400,7 @@ files:
 - lib/conceptql/behaviors/utilizable.rb
 - lib/conceptql/behaviors/windowable.rb
 - lib/conceptql/cli.rb
+- lib/conceptql/code_case_mismatch_error.rb
 - lib/conceptql/columnizer.rb
 - lib/conceptql/data_model.rb
 - lib/conceptql/data_model/base.rb
@@ -350,6 +409,19 @@ files:
 - lib/conceptql/database.rb
 - lib/conceptql/date_adjuster.rb
 - lib/conceptql/diagram.rb
+- lib/conceptql/diagram_renderer.rb
+- lib/conceptql/diagram_renderer/app.rb
+- lib/conceptql/diagram_renderer/configuration.rb
+- lib/conceptql/diagram_renderer/diagram_data.rb
+- lib/conceptql/diagram_renderer/drawing.rb
+- lib/conceptql/diagram_renderer/geometry.rb
+- lib/conceptql/diagram_renderer/input_error.rb
+- lib/conceptql/diagram_renderer/js_format.rb
+- lib/conceptql/diagram_renderer/resvg.rb
+- lib/conceptql/diagram_renderer/sprite.rb
+- lib/conceptql/diagram_renderer/static.rb
+- lib/conceptql/diagram_renderer/svg.rb
+- lib/conceptql/diagram_renderer/tree_layout.rb
 - lib/conceptql/lexicon.rb
 - lib/conceptql/lexicon/lexicon_gdm.rb
 - lib/conceptql/lexicon/lexicon_no_db.rb
@@ -449,6 +521,7 @@ files:
 homepage: https://github.com/outcomesinsights/conceptql
 licenses:
 - MIT
+- OFL-1.1
 metadata:
   rubygems_mfa_required: 'true'
 rdoc_options: []