conceptql 0.4.1 → 0.6.0

data/lib/conceptql/query_modifiers/gdm/provider_query_modifier.rb

@@ -29,20 +29,20 @@ module ConceptQL
                    Sequel[:p][:specialty_concept_id].as(:practitioner_specialty_concept_id)
                  )
           else
+            # The data model names the columns of the source table. ds.columns
+            # asked the database for them while the SQL was built (bead
+            # conceptql-6k8).
+            cols = dm.table_cols(source_table)
             ds = query.select_all
-            ds = if ds.columns.include?(:provider_id)
+            ds = if cols.include?(:provider_id)
                    ds
-            elsif dm.table_cols(source_table).include?(:practitioner_id)
+            elsif cols.include?(:practitioner_id)
                    ds.select_append(Sequel[:practitioner_id].as(:provider_id))
             else
                    ds.select_append(Sequel.cast(nil, :Bigint).as(:provider_id))
             end
             %i[role_type_concept_id context_specialty_concept_id practitioner_specialty_concept_id].each do |col|
-              ds = if ds.columns.include?(col)
-                     ds
-              else
-                     ds.select_append(Sequel.cast(nil, :Bigint).as(col))
-              end
+              ds = ds.select_append(Sequel.cast(nil, :Bigint).as(col)) unless cols.include?(col)
             end
             ds
           end.from_self

data/lib/conceptql/rdbms/generic.rb

@@ -53,9 +53,13 @@ module ConceptQL
           .inject(:+)
       end
 
+      # A NULL part is an empty string. The || operator gives NULL when one
+      # part is NULL, and a NULL uuid matches nothing (bead conceptql-18n).
+      # COALESCE keeps the place of each part, so "1//t/2008-01-01/" cannot
+      # be read as other parts. Postgres, DuckDB and Spark all support it.
       def uuid_items
         %w[person_id criterion_id criterion_table start_date end_date].map do |column|
-          Sequel.cast_string(column.to_sym)
+          Sequel.function(:coalesce, Sequel.cast_string(column.to_sym), "")
         end
       end
 
@@ -79,6 +83,22 @@ module ConceptQL
         false
       end
 
+      # True if an IN subquery may sit in a SELECT-list expression, such as
+      # the CASE flags of One In Two Out. Scope#with_ctes reads a long IN list
+      # from its list CTE there only when this is true.
+      def supports_in_subquery_in_select?
+        true
+      end
+
+      # True if a NOT IN subquery may sit in a nested condition, such as the
+      # "IS NULL OR NOT IN" of the gender operator. Scope#with_ctes reads a
+      # long NOT IN list from its list CTE only when this is true. It does not
+      # check whether a NOT IN condition is nested, so false keeps every long
+      # NOT IN list a NOT IN list.
+      def supports_nested_not_in_subquery?
+        true
+      end
+
       def create_options(_scope, _ds)
         opts = {}
         opts[:analyze] = opts[:explain] = explain_temp_tables?

data/lib/conceptql/rdbms/spark.rb

@@ -13,6 +13,18 @@ module ConceptQL
       def null_safe_equal(lhs, rhs)
         Sequel.lit("(? <=> ?)", lhs, rhs)
       end
+
+      # Spark accepts an IN subquery only in a filter or a join: "IN/EXISTS
+      # predicate sub-queries can only be used in Filter/Join" (Spark 3.3).
+      def supports_in_subquery_in_select?
+        false
+      end
+
+      # Spark rejects a NOT IN subquery in a nested condition: "Null-aware
+      # predicate sub-queries cannot be used in nested conditions" (Spark 3.3).
+      def supports_nested_not_in_subquery?
+        false
+      end
     end
   end
 end

data/lib/conceptql/scope.rb

@@ -86,6 +86,10 @@ module ConceptQL
 
     attr_accessor :person_ids
 
+    # The OneQueryCounts that Operator#annotate reads its counts from, or nil.
+    # Query#annotate sets it.
+    attr_accessor :one_query_counts
+
     attr_reader :known_operators, :recall_stack, :recall_dependencies, :annotation, :opts, :query_columns
 
     def initialize(opts = {})
@@ -93,6 +97,7 @@ module ConceptQL
       @recall_dependencies = {}
       @recall_stack = []
       @label_cte_names = {}
+      @operator_cte_names = {}.compare_by_identity
       @annotation = {}
       @opts = opts.dup
       @annotation[:errors] = @errors = {}
@@ -124,6 +129,16 @@ module ConceptQL
       c[domain] = counts
     end
 
+    # Operator#evaluate records the name of each CTE that holds an operator's
+    # rows. An operator evaluated more than once has more than one name.
+    def record_cte(op, name)
+      (@operator_cte_names[op] ||= []) << name
+    end
+
+    def cte_names(op)
+      @operator_cte_names.fetch(op, [])
+    end
+
     def add_required_columns(op)
       @query_columns |= op.required_columns if op.required_columns
     end
@@ -242,24 +257,70 @@ module ConceptQL
       true
     end
 
+    # An IN list of at least this many integers (concept IDs, mostly) goes to
+    # the database once, as a CTE that each copy of the list reads
+    # (concept_list_condition). Postgres spends planning time on each copy of
+    # a long IN list: one 111-operator algorithm spent 485 ms planning 52
+    # copies of 2,442-item provenance lists, and 156 ms with one CTE for each
+    # list. A shorter list stays an IN list, because a list CTE can change the
+    # Postgres plan for the worse: with a 231-item list as a CTE, the count
+    # statement of a test statement ran in 36 ms instead of 14 ms.
+    CONCEPT_LIST_MIN_ITEMS = 1000
+
+    # Takes the CTEs out of the datasets in an expression, and replaces its
+    # long IN lists with concept lists. With datasets: false, it leaves the
+    # datasets as they are (for a SELECT list). With concept_lists: false, it
+    # leaves the IN lists as they are. With not_in_lists: false, it leaves the
+    # NOT IN lists as they are.
     class CteExtractor < Sequel::ASTTransformer
-      def initialize(scope, ctes)
+      def initialize(scope, ctes, db, datasets: true, concept_lists: true, not_in_lists: true)
         super()
         @scope = scope
         @ctes = ctes
+        @db = db
+        @datasets = datasets
+        @concept_lists = concept_lists
+        @not_in_lists = not_in_lists
       end
 
       private
 
       def v(o)
         if o.is_a?(Sequel::Dataset)
-          @scope.recursive_extract_ctes(o, @ctes)
+          @datasets ? @scope.recursive_extract_ctes(o, @ctes) : o
+        elsif @concept_lists && @scope.concept_list?(o) && (@not_in_lists || o.op == :IN)
+          @scope.concept_list_condition(o, @db, @ctes, v(o.args.first))
         else
           super
         end
       end
     end
 
+    # True for an IN or NOT IN condition on a list of at least
+    # CONCEPT_LIST_MIN_ITEMS integers.
+    def concept_list?(expr)
+      expr.is_a?(Sequel::SQL::BooleanExpression) && %i[IN NOT\ IN].include?(expr.op) &&
+        (list = expr.args[1]).is_a?(Array) && list.length >= CONCEPT_LIST_MIN_ITEMS && list.all?(Integer)
+    end
+
+    # expr (a concept_list? condition) with its list replaced by a subquery
+    # of the CTE that holds the list. Each distinct list has one CTE in the
+    # scope, which joins ctes. The rows are the same: the list has no NULL.
+    def concept_list_condition(expr, db, ctes, column)
+      ids = expr.args[1]
+      name = (@concept_lists ||= {})[ids] ||= cte_name(:concept_list)
+      ctes << [ name, concept_list_dataset(db, ids), @concept_list_options || {} ]
+      Sequel::SQL::BooleanExpression.new(expr.op, column, db.from(name).select(:concept_id))
+    end
+
+    # One concept_id column with a row for each ID, in SQL that Postgres,
+    # DuckDB and Spark all accept.
+    def concept_list_dataset(db, ids)
+      rows = ids.map { |id| "(#{db.literal(id)})" }.join(", ")
+      alias_sql = "#{db.literal(Sequel.identifier(:t))}(#{db.literal(Sequel.identifier(:concept_id))})"
+      db.dataset.with_sql("SELECT * FROM (VALUES #{rows}) AS #{alias_sql}")
+    end
+
     def recursive_extract_cte_expr(t, ctes)
       case t
       when Sequel::Dataset
@@ -314,8 +375,21 @@ module ConceptQL
         # p [:rec_compounds, ctes.map(&:first), compounds]
       end
 
+      # Spark rejects a NOT IN subquery in a nested condition, so there a
+      # NOT IN list stays a NOT IN list. @not_in_concept_lists and
+      # @select_concept_lists are nil when with_ctes did not run (a direct
+      # call), and nil rewrites.
+      not_in_lists = @not_in_concept_lists != false
       if (where = query.opts[:where])
-        query = query.clone(where: CteExtractor.new(self, ctes).transform(where))
+        query = query.clone(where: CteExtractor.new(self, ctes, query.db, not_in_lists: not_in_lists).transform(where))
+      end
+
+      # One In Two Out flags its events with CASE expressions on the same
+      # provenance lists as its WHERE. Spark rejects an IN subquery in a
+      # SELECT list, so there the lists stay IN lists.
+      if (select = query.opts[:select])
+        extractor = CteExtractor.new(self, ctes, query.db, datasets: false, concept_lists: @select_concept_lists != false, not_in_lists: not_in_lists)
+        query = query.clone(select: extractor.transform(select))
       end
 
       query
@@ -335,39 +409,122 @@ module ConceptQL
     # reads before the dataset itself, so an order that is already correct
     # does not change. (sort_ctes sorts by layers, and would move CTEs.)
     def sort_extracted_ctes(ctes)
-      unsorted = ctes.uniq(&:first)
-      literals = unsorted.to_h { |name, ds| [ name, ds.db.literal(name) ] }
-      deps = unsorted.to_h do |name, ds|
-        sql = ds.sql
-        [ name, literals.filter_map { |other, literal| other if other != name && sql.include?(literal) } ]
+      sorted_ctes_with_sql(ctes).map(&:entry)
+    end
+
+    # One extracted CTE in sorted_ctes_with_sql: its [name, dataset, WITH
+    # options] entry, its name as the adapter quotes it, its SQL, and the
+    # quoted names of the CTEs that its SQL reads.
+    SortedCte = Struct.new(:entry, :literal, :sql, :reads)
+
+    # The work of sort_extracted_ctes. It keeps each CTE's SQL, so that
+    # merge_identical_ctes does not build it again. The names are compared as
+    # quoted strings: a comparison of two Sequel identifiers is much slower.
+    def sorted_ctes_with_sql(ctes)
+      unsorted = ctes.uniq(&:first).map do |entry|
+        name, ds = entry
+        SortedCte.new(entry, ds.db.literal(name), ds.sql)
+      end
+      literals = unsorted.map(&:literal)
+      unsorted.each do |cte|
+        cte.reads = literals.select { |literal| !literal.equal?(cte.literal) && cte.sql.include?(literal) }
       end
 
       sorted = []
+      placed = Set.new
       until unsorted.empty?
-        placed = sorted.map(&:first)
-        index = unsorted.index { |name, _| (deps[name] - placed).empty? }
-        raise(CteCycleError, unsorted.map { |name, _| literals[name] }) unless index
+        index = unsorted.index { |cte| cte.reads.all? { |literal| placed.include?(literal) } }
+        raise(CteCycleError, unsorted.map(&:literal)) unless index
 
-        sorted << unsorted.delete_at(index)
+        cte = unsorted.delete_at(index)
+        placed << cte.literal
+        sorted << cte
       end
       sorted
     end
 
-    def with_ctes(op, db, options = {})
+    # The opts key of the query that with_ctes returns. Its value maps the
+    # name of each CTE that merge_identical_ctes removed to the name of the
+    # CTE that the query reads in its place.
+    MERGED_CTES = :conceptql_merged_ctes
+
+    # Sorts the extracted CTEs (sort_extracted_ctes) and keeps one CTE of each
+    # group of CTEs with the same SQL and WITH options. A statement evaluates
+    # some operators more than once (a Recall, or an operator that reads its
+    # upstream twice), and each evaluation gives its CTE a new name. Each
+    # reader of a removed CTE reads the kept one instead. The CTEs come in
+    # dependency order, so a CTE's SQL is compared after the CTEs it reads are
+    # renamed. Thus two CTEs that read two copies of one CTE merge too.
+    #
+    # The CTEs that root_sql reads are never removed, because the root query
+    # is a dataset, not SQL text, and its names are not renamed.
+    #
+    # Each kept CTE's dataset becomes its SQL text (Dataset#with_sql), so the
+    # final query does not build that SQL again.
+    #
+    # Returns the kept [name, dataset, WITH options] entries in dependency
+    # order, and a Hash of each removed name to the name that replaces it.
+    def merge_identical_ctes(ctes, root_sql)
+      renames = {}
+      merged = {}
+      kept = {}
+      pattern = nil
+      pattern_size = 0
+      entries = sorted_ctes_with_sql(ctes).filter_map do |cte|
+        name, ds, with_opts = cte.entry
+        sql = cte.sql
+        if cte.reads.any? { |literal| renames.key?(literal) }
+          unless pattern_size == renames.length
+            pattern = Regexp.union(renames.keys)
+            pattern_size = renames.length
+          end
+          sql = sql.gsub(pattern, renames)
+        end
+
+        key = [ sql, with_opts ]
+        if (same = kept[key]) && !root_sql.include?(cte.literal)
+          renames[cte.literal] = same.literal
+          merged[name] = same.entry.first
+          next
+        end
+
+        kept[key] ||= cte
+        [ name, ds.db.dataset.with_sql(sql), with_opts ]
+      end
+      [ entries, merged ]
+    end
+
+    # The CTEs of each dataset in extra (an evaluated operator) join the WITH
+    # list too, so the query can read operators that op does not evaluate.
+    def with_ctes(op, db, options = {}, extra: [])
       raise "recall operator use without matching label" unless valid?
 
       query = op.evaluate(db)
+      extra.each { |ds| query = query.clone(with: query.opts[:with] + ds.opts[:with]) }
       rdbms = op.rdbms
+      # A concept list is read by many CTEs, so it runs once.
+      @concept_list_options = rdbms.supports_materialized? ? { materialized: true } : {}
+      @select_concept_lists = rdbms.supports_in_subquery_in_select?
+      @not_in_concept_lists = rdbms.supports_nested_not_in_subquery?
 
       query = query.from_self
       temp_tables = []
-      ctes.each do |label, operator|
-        temp_tables << [ label_cte_name(label), recursive_extract_ctes(operator.evaluate(db), temp_tables) ]
+      # With CONCEPTQL_AVOID_CTES, each operator is a subquery
+      # (Operator#evaluate) and Scope#from inlines each labeled operator, so
+      # nothing is extracted and a long IN list stays an IN list. Only the
+      # shared tables (add_shared_table) remain.
+      if ConceptQL.avoid_ctes?
+        temp_tables.concat(@shared_tables || [])
+      else
+        ctes.each do |label, operator|
+          temp_tables << [ label_cte_name(label), recursive_extract_ctes(operator.evaluate(db), temp_tables) ]
+        end
+        query = recursive_extract_ctes(query, temp_tables)
       end
-      query = recursive_extract_ctes(query, temp_tables)
       shared_tables, temp_tables = temp_tables.partition { |_, _, with_opts| with_opts&.fetch(SHARED_TABLE, false) }
       shared_tables = shared_tables.uniq(&:first)
-      temp_tables = sort_extracted_ctes(temp_tables)
+      temp_tables, merged = merge_identical_ctes(temp_tables, query.sql)
+      query = query.clone(MERGED_CTES => merged)
 
       if force_temp_tables?(options)
         scope = self
@@ -425,13 +582,11 @@ module ConceptQL
           end
         end
       else
-        unless ConceptQL.avoid_ctes?
-          temp_tables.each do |table_name, ds, with_opts|
-            query = if with_opts.nil? || with_opts.empty?
-                      query.with(table_name, ds)
-            else
-                      query.with(table_name, ds, with_opts)
-            end
+        temp_tables.each do |table_name, ds, with_opts|
+          query = if with_opts.nil? || with_opts.empty?
+                    query.with(table_name, ds)
+          else
+                    query.with(table_name, ds, with_opts)
           end
         end
 
@@ -449,6 +604,14 @@ module ConceptQL
       shared_tables.empty? ? query : create_shared_tables_on_read(query, shared_tables, op)
     end
 
+    # With CONCEPTQL_AVOID_CTES, Behaviors::Utilizable#evaluate records each
+    # shared table here, because with_ctes extracts no WITH entry to find it
+    # in. Returns a dataset that reads the table.
+    def add_shared_table(db, name, list)
+      (@shared_tables ||= []) << [ name, list, { SHARED_TABLE => true } ]
+      db[name]
+    end
+
     # The sql_statements entries that create the shared tables that do not
     # exist yet. They come first, because the other entries read them.
     def shared_table_statements(shared_tables, op)

data/lib/conceptql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConceptQL
-  VERSION = "0.4.1"
+  VERSION = "0.6.0"
 end

data/lib/conceptql/vocabularies/drg_systems.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "date"
+
+module ConceptQL
+  module Vocabularies
+    # The two DRG systems and their operators, ms_drg and cms_drg (bead
+    # conceptql-alw, seed conceptql-7p3).
+    #
+    # CMS replaced the CMS-DRG system with MS-DRG on 2007-10-01 and reused
+    # most DRG codes for new meanings: DRG 010 meant "Nervous System
+    # Neoplasms" before that day and "Pancreas transplant" after it. One
+    # vocabulary, DRG, holds both systems, and an event row does not record
+    # which meaning applied: the ETL stores one concept ID for each source code
+    # whatever the event date. So a statement names the system, and the system
+    # decides by the event date. ["ms_drg", "010"] selects the DRG 010 events
+    # from 2007-10-01, and ["cms_drg", "010"] selects the DRG 010 events before
+    # that day. The concept-ID lookup is the one that drg uses, so the
+    # operators work with the OHDSI and the GDM lexicons alike. They use no
+    # OHDSI validity date and no concept class to select events.
+    #
+    # Each operator is a vocabulary operator of vocabulary DRG. So its code
+    # format is the DRG row of vocabulary_formats (a vocabulary operator uses
+    # the row of the vocabulary it selects), and its unknown-code warning
+    # checks the DRG vocabulary.
+    #
+    # The deprecated drg operator selects the events of both systems.
+    module DrgSystems
+      # The first day of MS-DRG. ms_drg selects the events on or after this
+      # day, and cms_drg the events before it.
+      MS_DRG_START_DATE = Date.new(2007, 10, 1)
+
+      # The vocabulary and the operator that hold both systems.
+      VOCABULARY_ID = "DRG"
+      DRG_OPERATOR = "drg"
+
+      DRG_DEPRECATION = "drg will become an error in a later release. A DRG code can have two meanings, " \
+                        "and drg selects the events of both. Use ms_drg for the MS-DRG meaning (events from " \
+                        "#{MS_DRG_START_DATE.iso8601}) or cms_drg for the CMS-DRG meaning (events before " \
+                        "#{MS_DRG_START_DATE.iso8601}).".freeze
+
+      # operator: the operator name. concept_class_id: the OHDSI concept class
+      # of the system's meanings. from_cutover: true when the system starts on
+      # MS_DRG_START_DATE, false when it ends the day before.
+      System = Data.define(:operator, :name, :long_name, :concept_class_id, :from_cutover) do
+        # The condition on the event's start date.
+        def date_condition(start_date_column)
+          column = Sequel.identifier(start_date_column)
+          from_cutover ? column >= MS_DRG_START_DATE : column < MS_DRG_START_DATE
+        end
+      end
+
+      SYSTEMS = [
+        System.new(operator: "ms_drg", name: "MS-DRG",
+                   long_name: "Medicare Severity Diagnosis-Related Group (CMS), events from " \
+                              "#{MS_DRG_START_DATE.iso8601}",
+                   concept_class_id: "MS-DRG", from_cutover: true),
+        System.new(operator: "cms_drg", name: "CMS-DRG",
+                   long_name: "CMS Diagnosis-Related Group, events before #{MS_DRG_START_DATE.iso8601}",
+                   concept_class_id: "DRG", from_cutover: false)
+      ].to_h { |system| [ system.operator, system ] }.freeze
+
+      class << self
+        def system(operator)
+          SYSTEMS[operator]
+        end
+
+        # The vocabulary entries of ms_drg and cms_drg, as id => Entry. Each
+        # copies the entry of drg (its domain, for example) and renames it.
+        # Without a drg entry there is none.
+        def entries(drg_entry)
+          return {} unless drg_entry
+
+          SYSTEMS.transform_values do |system|
+            drg_entry.merge(Entry.new(id: system.operator,
+                                      omopv5_vocabulary_id: VOCABULARY_ID,
+                                      preferred_name: system.name,
+                                      vocabulary_short_name: system.name,
+                                      vocabulary_long_name: system.long_name))
+          end
+        end
+      end
+
+      # The behavior of the ms_drg and cms_drg operator classes.
+      module OperatorBehavior
+        def drg_system
+          DrgSystems.system(vocab_entry.id)
+        end
+
+        # Each code is named by its meaning in the operator's system when the
+        # lexicon can tell: the OHDSI vocabulary gives each meaning a concept
+        # class (MS-DRG or DRG). A code without a row of that class, and every
+        # code under a lexicon that keeps no class (GDM), gets every name of
+        # the code, as drg gives.
+        def describe_codes(db, codes)
+          return super if select_all?
+
+          in_system = dm.concepts_to_codes_in_class(db, vocabulary_id, drg_system.concept_class_id, codes)
+          return super if in_system.blank?
+
+          named = in_system.map { |code, _| code.downcase }
+          rest = codes.reject { |code| named.include?(code.downcase) }
+          in_system + (rest.empty? ? [] : super(db, rest))
+        end
+
+        # The operator selects one meaning of each code, so a code that the
+        # vocabulary reused across the two systems needs no warning.
+        def warn_about_reused_codes(_db, _codes); end
+
+        private
+
+        def where_clause(db)
+          Sequel.&(super, drg_system.date_condition(dm.start_date_column(db, criterion_table)))
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/dynamic_vocabularies.rb

@@ -2,6 +2,7 @@
 
 require "csv"
 require_relative "entry"
+require_relative "drg_systems"
 
 module ConceptQL
   module Vocabularies
@@ -26,11 +27,16 @@ module ConceptQL
         end
       end
 
+      # Every vocabulary entry, as id => Entry, and the entries of ms_drg and
+      # cms_drg, which copy the entry of drg (DrgSystems).
       def all_vocabs
-        @all_vocabs ||= each_vocab.each_with_object({}) do |row, h|
-          entry = Entry.new(row.to_hash.compact)
-          h[entry.id] ||= entry
-          h[entry.id] = h[entry.id].merge(entry)
+        @all_vocabs ||= begin
+          vocabs = each_vocab.each_with_object({}) do |row, h|
+            entry = Entry.new(row.to_hash.compact)
+            h[entry.id] ||= entry
+            h[entry.id] = h[entry.id].merge(entry)
+          end
+          vocabs.merge(DrgSystems.entries(vocabs[DrgSystems::DRG_OPERATOR]))
         end
       end
 

data/lib/conceptql/vocabularies/entry.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "formats"
 require_relative "vocabulary_class_factory"
 
 module ConceptQL
@@ -9,12 +10,10 @@ module ConceptQL
 
       METHODS = %i[
         id
-        omopv4_vocabulary_id
         omopv5_vocabulary_id
         domain
         vocabulary_short_name
         vocabulary_long_name
-        format_regexp
       ].freeze
 
       METHODS.each do |meth|
@@ -24,15 +23,11 @@ module ConceptQL
       end
 
       METADATA_METHODS = METHODS + %i[
+        format_regexp
         preferred_name
         predominant_domains
       ]
 
-      COST_RELATED_VOCABS = [
-        "Revenue Code",
-        "DRG"
-      ].freeze
-
       def initialize(hash)
         @hash = hash
         @hash[:omopv5_vocabulary_id] ||= @hash[:id]
@@ -54,18 +49,20 @@ module ConceptQL
         self.class.new(hash.merge(other_entry.hash))
       end
 
-      def belongs_in_omopv4_plus?
-        @belongs_in_omopv4_plus ||= (!from_lexicon? || from_csv?) \
-          && has_domain? \
-          && visible?
-      end
-
       def to_hash
         METADATA_METHODS.to_h do |meth|
           [ meth, send(meth) ]
         end
       end
 
+      # The anchored code format from vocabulary_formats, as a string, or nil
+      # when vocabulary_formats has no row for this vocabulary. A
+      # format_regexp column in a vocabularies CSV is ignored: vocabulary_formats
+      # is the only source (bead conceptql-bns).
+      def format_regexp
+        Formats.pattern(omopv5_id)
+      end
+
       def short_name
         vocabulary_short_name || vocabulary_long_name
       end
@@ -74,16 +71,19 @@ module ConceptQL
         vocabulary_long_name || vocabulary_short_name
       end
 
+      # A preferred_name in the hash wins: ms_drg and cms_drg select vocabulary
+      # DRG, but each is named after its own DRG system.
       def preferred_name
-        omopv5_id || id
+        hash[:preferred_name] || omopv5_id || id
       end
 
       def aliases
         @hash[:aliases_arr]
       end
 
+      # A vocabulary without a domain selects condition_occurrence records
       def predominant_domains
-        Array(domain || :condition_occurrence).flatten
+        Array(domain || "condition_occurrence").flatten
       end
 
       def from_lexicon?
@@ -114,10 +114,6 @@ module ConceptQL
         hash[:domain] =~ /drug_exposure/i
       end
 
-      def is_costish?
-        belongs_in_omopv4_plus? && COST_RELATED_VOCABS.include?(omopv5_vocabulary_id)
-      end
-
       def visible?
         !hidden?
       end
@@ -126,10 +122,6 @@ module ConceptQL
         omopv5_vocabulary_id || id
       end
 
-      def omopv4_id
-        omopv4_vocabulary_id
-      end
-
       def get_klasses
         VocabularyClassFactory.new(self).get_klasses
       end