conceptql 0.5.0 → 0.6.0

data/lib/conceptql/scope.rb

@@ -86,6 +86,10 @@ module ConceptQL
 
     attr_accessor :person_ids
 
+    # The OneQueryCounts that Operator#annotate reads its counts from, or nil.
+    # Query#annotate sets it.
+    attr_accessor :one_query_counts
+
     attr_reader :known_operators, :recall_stack, :recall_dependencies, :annotation, :opts, :query_columns
 
     def initialize(opts = {})
@@ -93,6 +97,7 @@ module ConceptQL
       @recall_dependencies = {}
       @recall_stack = []
       @label_cte_names = {}
+      @operator_cte_names = {}.compare_by_identity
       @annotation = {}
       @opts = opts.dup
       @annotation[:errors] = @errors = {}
@@ -124,6 +129,16 @@ module ConceptQL
       c[domain] = counts
     end
 
+    # Operator#evaluate records the name of each CTE that holds an operator's
+    # rows. An operator evaluated more than once has more than one name.
+    def record_cte(op, name)
+      (@operator_cte_names[op] ||= []) << name
+    end
+
+    def cte_names(op)
+      @operator_cte_names.fetch(op, [])
+    end
+
     def add_required_columns(op)
       @query_columns |= op.required_columns if op.required_columns
     end
@@ -242,24 +257,70 @@ module ConceptQL
       true
     end
 
+    # An IN list of at least this many integers (concept IDs, mostly) goes to
+    # the database once, as a CTE that each copy of the list reads
+    # (concept_list_condition). Postgres spends planning time on each copy of
+    # a long IN list: one 111-operator algorithm spent 485 ms planning 52
+    # copies of 2,442-item provenance lists, and 156 ms with one CTE for each
+    # list. A shorter list stays an IN list, because a list CTE can change the
+    # Postgres plan for the worse: with a 231-item list as a CTE, the count
+    # statement of a test statement ran in 36 ms instead of 14 ms.
+    CONCEPT_LIST_MIN_ITEMS = 1000
+
+    # Takes the CTEs out of the datasets in an expression, and replaces its
+    # long IN lists with concept lists. With datasets: false, it leaves the
+    # datasets as they are (for a SELECT list). With concept_lists: false, it
+    # leaves the IN lists as they are. With not_in_lists: false, it leaves the
+    # NOT IN lists as they are.
     class CteExtractor < Sequel::ASTTransformer
-      def initialize(scope, ctes)
+      def initialize(scope, ctes, db, datasets: true, concept_lists: true, not_in_lists: true)
         super()
         @scope = scope
         @ctes = ctes
+        @db = db
+        @datasets = datasets
+        @concept_lists = concept_lists
+        @not_in_lists = not_in_lists
       end
 
       private
 
       def v(o)
         if o.is_a?(Sequel::Dataset)
-          @scope.recursive_extract_ctes(o, @ctes)
+          @datasets ? @scope.recursive_extract_ctes(o, @ctes) : o
+        elsif @concept_lists && @scope.concept_list?(o) && (@not_in_lists || o.op == :IN)
+          @scope.concept_list_condition(o, @db, @ctes, v(o.args.first))
         else
           super
         end
       end
     end
 
+    # True for an IN or NOT IN condition on a list of at least
+    # CONCEPT_LIST_MIN_ITEMS integers.
+    def concept_list?(expr)
+      expr.is_a?(Sequel::SQL::BooleanExpression) && %i[IN NOT\ IN].include?(expr.op) &&
+        (list = expr.args[1]).is_a?(Array) && list.length >= CONCEPT_LIST_MIN_ITEMS && list.all?(Integer)
+    end
+
+    # expr (a concept_list? condition) with its list replaced by a subquery
+    # of the CTE that holds the list. Each distinct list has one CTE in the
+    # scope, which joins ctes. The rows are the same: the list has no NULL.
+    def concept_list_condition(expr, db, ctes, column)
+      ids = expr.args[1]
+      name = (@concept_lists ||= {})[ids] ||= cte_name(:concept_list)
+      ctes << [ name, concept_list_dataset(db, ids), @concept_list_options || {} ]
+      Sequel::SQL::BooleanExpression.new(expr.op, column, db.from(name).select(:concept_id))
+    end
+
+    # One concept_id column with a row for each ID, in SQL that Postgres,
+    # DuckDB and Spark all accept.
+    def concept_list_dataset(db, ids)
+      rows = ids.map { |id| "(#{db.literal(id)})" }.join(", ")
+      alias_sql = "#{db.literal(Sequel.identifier(:t))}(#{db.literal(Sequel.identifier(:concept_id))})"
+      db.dataset.with_sql("SELECT * FROM (VALUES #{rows}) AS #{alias_sql}")
+    end
+
     def recursive_extract_cte_expr(t, ctes)
       case t
       when Sequel::Dataset
@@ -314,8 +375,21 @@ module ConceptQL
         # p [:rec_compounds, ctes.map(&:first), compounds]
       end
 
+      # Spark rejects a NOT IN subquery in a nested condition, so there a
+      # NOT IN list stays a NOT IN list. @not_in_concept_lists and
+      # @select_concept_lists are nil when with_ctes did not run (a direct
+      # call), and nil rewrites.
+      not_in_lists = @not_in_concept_lists != false
       if (where = query.opts[:where])
-        query = query.clone(where: CteExtractor.new(self, ctes).transform(where))
+        query = query.clone(where: CteExtractor.new(self, ctes, query.db, not_in_lists: not_in_lists).transform(where))
+      end
+
+      # One In Two Out flags its events with CASE expressions on the same
+      # provenance lists as its WHERE. Spark rejects an IN subquery in a
+      # SELECT list, so there the lists stay IN lists.
+      if (select = query.opts[:select])
+        extractor = CteExtractor.new(self, ctes, query.db, datasets: false, concept_lists: @select_concept_lists != false, not_in_lists: not_in_lists)
+        query = query.clone(select: extractor.transform(select))
       end
 
       query
@@ -335,39 +409,122 @@ module ConceptQL
     # reads before the dataset itself, so an order that is already correct
     # does not change. (sort_ctes sorts by layers, and would move CTEs.)
     def sort_extracted_ctes(ctes)
-      unsorted = ctes.uniq(&:first)
-      literals = unsorted.to_h { |name, ds| [ name, ds.db.literal(name) ] }
-      deps = unsorted.to_h do |name, ds|
-        sql = ds.sql
-        [ name, literals.filter_map { |other, literal| other if other != name && sql.include?(literal) } ]
+      sorted_ctes_with_sql(ctes).map(&:entry)
+    end
+
+    # One extracted CTE in sorted_ctes_with_sql: its [name, dataset, WITH
+    # options] entry, its name as the adapter quotes it, its SQL, and the
+    # quoted names of the CTEs that its SQL reads.
+    SortedCte = Struct.new(:entry, :literal, :sql, :reads)
+
+    # The work of sort_extracted_ctes. It keeps each CTE's SQL, so that
+    # merge_identical_ctes does not build it again. The names are compared as
+    # quoted strings: a comparison of two Sequel identifiers is much slower.
+    def sorted_ctes_with_sql(ctes)
+      unsorted = ctes.uniq(&:first).map do |entry|
+        name, ds = entry
+        SortedCte.new(entry, ds.db.literal(name), ds.sql)
+      end
+      literals = unsorted.map(&:literal)
+      unsorted.each do |cte|
+        cte.reads = literals.select { |literal| !literal.equal?(cte.literal) && cte.sql.include?(literal) }
       end
 
       sorted = []
+      placed = Set.new
       until unsorted.empty?
-        placed = sorted.map(&:first)
-        index = unsorted.index { |name, _| (deps[name] - placed).empty? }
-        raise(CteCycleError, unsorted.map { |name, _| literals[name] }) unless index
+        index = unsorted.index { |cte| cte.reads.all? { |literal| placed.include?(literal) } }
+        raise(CteCycleError, unsorted.map(&:literal)) unless index
 
-        sorted << unsorted.delete_at(index)
+        cte = unsorted.delete_at(index)
+        placed << cte.literal
+        sorted << cte
       end
       sorted
     end
 
-    def with_ctes(op, db, options = {})
+    # The opts key of the query that with_ctes returns. Its value maps the
+    # name of each CTE that merge_identical_ctes removed to the name of the
+    # CTE that the query reads in its place.
+    MERGED_CTES = :conceptql_merged_ctes
+
+    # Sorts the extracted CTEs (sort_extracted_ctes) and keeps one CTE of each
+    # group of CTEs with the same SQL and WITH options. A statement evaluates
+    # some operators more than once (a Recall, or an operator that reads its
+    # upstream twice), and each evaluation gives its CTE a new name. Each
+    # reader of a removed CTE reads the kept one instead. The CTEs come in
+    # dependency order, so a CTE's SQL is compared after the CTEs it reads are
+    # renamed. Thus two CTEs that read two copies of one CTE merge too.
+    #
+    # The CTEs that root_sql reads are never removed, because the root query
+    # is a dataset, not SQL text, and its names are not renamed.
+    #
+    # Each kept CTE's dataset becomes its SQL text (Dataset#with_sql), so the
+    # final query does not build that SQL again.
+    #
+    # Returns the kept [name, dataset, WITH options] entries in dependency
+    # order, and a Hash of each removed name to the name that replaces it.
+    def merge_identical_ctes(ctes, root_sql)
+      renames = {}
+      merged = {}
+      kept = {}
+      pattern = nil
+      pattern_size = 0
+      entries = sorted_ctes_with_sql(ctes).filter_map do |cte|
+        name, ds, with_opts = cte.entry
+        sql = cte.sql
+        if cte.reads.any? { |literal| renames.key?(literal) }
+          unless pattern_size == renames.length
+            pattern = Regexp.union(renames.keys)
+            pattern_size = renames.length
+          end
+          sql = sql.gsub(pattern, renames)
+        end
+
+        key = [ sql, with_opts ]
+        if (same = kept[key]) && !root_sql.include?(cte.literal)
+          renames[cte.literal] = same.literal
+          merged[name] = same.entry.first
+          next
+        end
+
+        kept[key] ||= cte
+        [ name, ds.db.dataset.with_sql(sql), with_opts ]
+      end
+      [ entries, merged ]
+    end
+
+    # The CTEs of each dataset in extra (an evaluated operator) join the WITH
+    # list too, so the query can read operators that op does not evaluate.
+    def with_ctes(op, db, options = {}, extra: [])
       raise "recall operator use without matching label" unless valid?
 
       query = op.evaluate(db)
+      extra.each { |ds| query = query.clone(with: query.opts[:with] + ds.opts[:with]) }
       rdbms = op.rdbms
+      # A concept list is read by many CTEs, so it runs once.
+      @concept_list_options = rdbms.supports_materialized? ? { materialized: true } : {}
+      @select_concept_lists = rdbms.supports_in_subquery_in_select?
+      @not_in_concept_lists = rdbms.supports_nested_not_in_subquery?
 
       query = query.from_self
       temp_tables = []
-      ctes.each do |label, operator|
-        temp_tables << [ label_cte_name(label), recursive_extract_ctes(operator.evaluate(db), temp_tables) ]
+      # With CONCEPTQL_AVOID_CTES, each operator is a subquery
+      # (Operator#evaluate) and Scope#from inlines each labeled operator, so
+      # nothing is extracted and a long IN list stays an IN list. Only the
+      # shared tables (add_shared_table) remain.
+      if ConceptQL.avoid_ctes?
+        temp_tables.concat(@shared_tables || [])
+      else
+        ctes.each do |label, operator|
+          temp_tables << [ label_cte_name(label), recursive_extract_ctes(operator.evaluate(db), temp_tables) ]
+        end
+        query = recursive_extract_ctes(query, temp_tables)
       end
-      query = recursive_extract_ctes(query, temp_tables)
       shared_tables, temp_tables = temp_tables.partition { |_, _, with_opts| with_opts&.fetch(SHARED_TABLE, false) }
       shared_tables = shared_tables.uniq(&:first)
-      temp_tables = sort_extracted_ctes(temp_tables)
+      temp_tables, merged = merge_identical_ctes(temp_tables, query.sql)
+      query = query.clone(MERGED_CTES => merged)
 
       if force_temp_tables?(options)
         scope = self
@@ -425,13 +582,11 @@ module ConceptQL
           end
         end
       else
-        unless ConceptQL.avoid_ctes?
-          temp_tables.each do |table_name, ds, with_opts|
-            query = if with_opts.nil? || with_opts.empty?
-                      query.with(table_name, ds)
-            else
-                      query.with(table_name, ds, with_opts)
-            end
+        temp_tables.each do |table_name, ds, with_opts|
+          query = if with_opts.nil? || with_opts.empty?
+                    query.with(table_name, ds)
+          else
+                    query.with(table_name, ds, with_opts)
           end
         end
 
@@ -449,6 +604,14 @@ module ConceptQL
       shared_tables.empty? ? query : create_shared_tables_on_read(query, shared_tables, op)
     end
 
+    # With CONCEPTQL_AVOID_CTES, Behaviors::Utilizable#evaluate records each
+    # shared table here, because with_ctes extracts no WITH entry to find it
+    # in. Returns a dataset that reads the table.
+    def add_shared_table(db, name, list)
+      (@shared_tables ||= []) << [ name, list, { SHARED_TABLE => true } ]
+      db[name]
+    end
+
     # The sql_statements entries that create the shared tables that do not
     # exist yet. They come first, because the other entries read them.
     def shared_table_statements(shared_tables, op)

data/lib/conceptql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConceptQL
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/conceptql/vocabularies/drg_systems.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "date"
+
+module ConceptQL
+  module Vocabularies
+    # The two DRG systems and their operators, ms_drg and cms_drg (bead
+    # conceptql-alw, seed conceptql-7p3).
+    #
+    # CMS replaced the CMS-DRG system with MS-DRG on 2007-10-01 and reused
+    # most DRG codes for new meanings: DRG 010 meant "Nervous System
+    # Neoplasms" before that day and "Pancreas transplant" after it. One
+    # vocabulary, DRG, holds both systems, and an event row does not record
+    # which meaning applied: the ETL stores one concept ID for each source code
+    # whatever the event date. So a statement names the system, and the system
+    # decides by the event date. ["ms_drg", "010"] selects the DRG 010 events
+    # from 2007-10-01, and ["cms_drg", "010"] selects the DRG 010 events before
+    # that day. The concept-ID lookup is the one that drg uses, so the
+    # operators work with the OHDSI and the GDM lexicons alike. They use no
+    # OHDSI validity date and no concept class to select events.
+    #
+    # Each operator is a vocabulary operator of vocabulary DRG. So its code
+    # format is the DRG row of vocabulary_formats (a vocabulary operator uses
+    # the row of the vocabulary it selects), and its unknown-code warning
+    # checks the DRG vocabulary.
+    #
+    # The deprecated drg operator selects the events of both systems.
+    module DrgSystems
+      # The first day of MS-DRG. ms_drg selects the events on or after this
+      # day, and cms_drg the events before it.
+      MS_DRG_START_DATE = Date.new(2007, 10, 1)
+
+      # The vocabulary and the operator that hold both systems.
+      VOCABULARY_ID = "DRG"
+      DRG_OPERATOR = "drg"
+
+      DRG_DEPRECATION = "drg will become an error in a later release. A DRG code can have two meanings, " \
+                        "and drg selects the events of both. Use ms_drg for the MS-DRG meaning (events from " \
+                        "#{MS_DRG_START_DATE.iso8601}) or cms_drg for the CMS-DRG meaning (events before " \
+                        "#{MS_DRG_START_DATE.iso8601}).".freeze
+
+      # operator: the operator name. concept_class_id: the OHDSI concept class
+      # of the system's meanings. from_cutover: true when the system starts on
+      # MS_DRG_START_DATE, false when it ends the day before.
+      System = Data.define(:operator, :name, :long_name, :concept_class_id, :from_cutover) do
+        # The condition on the event's start date.
+        def date_condition(start_date_column)
+          column = Sequel.identifier(start_date_column)
+          from_cutover ? column >= MS_DRG_START_DATE : column < MS_DRG_START_DATE
+        end
+      end
+
+      SYSTEMS = [
+        System.new(operator: "ms_drg", name: "MS-DRG",
+                   long_name: "Medicare Severity Diagnosis-Related Group (CMS), events from " \
+                              "#{MS_DRG_START_DATE.iso8601}",
+                   concept_class_id: "MS-DRG", from_cutover: true),
+        System.new(operator: "cms_drg", name: "CMS-DRG",
+                   long_name: "CMS Diagnosis-Related Group, events before #{MS_DRG_START_DATE.iso8601}",
+                   concept_class_id: "DRG", from_cutover: false)
+      ].to_h { |system| [ system.operator, system ] }.freeze
+
+      class << self
+        def system(operator)
+          SYSTEMS[operator]
+        end
+
+        # The vocabulary entries of ms_drg and cms_drg, as id => Entry. Each
+        # copies the entry of drg (its domain, for example) and renames it.
+        # Without a drg entry there is none.
+        def entries(drg_entry)
+          return {} unless drg_entry
+
+          SYSTEMS.transform_values do |system|
+            drg_entry.merge(Entry.new(id: system.operator,
+                                      omopv5_vocabulary_id: VOCABULARY_ID,
+                                      preferred_name: system.name,
+                                      vocabulary_short_name: system.name,
+                                      vocabulary_long_name: system.long_name))
+          end
+        end
+      end
+
+      # The behavior of the ms_drg and cms_drg operator classes.
+      module OperatorBehavior
+        def drg_system
+          DrgSystems.system(vocab_entry.id)
+        end
+
+        # Each code is named by its meaning in the operator's system when the
+        # lexicon can tell: the OHDSI vocabulary gives each meaning a concept
+        # class (MS-DRG or DRG). A code without a row of that class, and every
+        # code under a lexicon that keeps no class (GDM), gets every name of
+        # the code, as drg gives.
+        def describe_codes(db, codes)
+          return super if select_all?
+
+          in_system = dm.concepts_to_codes_in_class(db, vocabulary_id, drg_system.concept_class_id, codes)
+          return super if in_system.blank?
+
+          named = in_system.map { |code, _| code.downcase }
+          rest = codes.reject { |code| named.include?(code.downcase) }
+          in_system + (rest.empty? ? [] : super(db, rest))
+        end
+
+        # The operator selects one meaning of each code, so a code that the
+        # vocabulary reused across the two systems needs no warning.
+        def warn_about_reused_codes(_db, _codes); end
+
+        private
+
+        def where_clause(db)
+          Sequel.&(super, drg_system.date_condition(dm.start_date_column(db, criterion_table)))
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/dynamic_vocabularies.rb

@@ -2,6 +2,7 @@
 
 require "csv"
 require_relative "entry"
+require_relative "drg_systems"
 
 module ConceptQL
   module Vocabularies
@@ -26,11 +27,16 @@ module ConceptQL
         end
       end
 
+      # Every vocabulary entry, as id => Entry, and the entries of ms_drg and
+      # cms_drg, which copy the entry of drg (DrgSystems).
       def all_vocabs
-        @all_vocabs ||= each_vocab.each_with_object({}) do |row, h|
-          entry = Entry.new(row.to_hash.compact)
-          h[entry.id] ||= entry
-          h[entry.id] = h[entry.id].merge(entry)
+        @all_vocabs ||= begin
+          vocabs = each_vocab.each_with_object({}) do |row, h|
+            entry = Entry.new(row.to_hash.compact)
+            h[entry.id] ||= entry
+            h[entry.id] = h[entry.id].merge(entry)
+          end
+          vocabs.merge(DrgSystems.entries(vocabs[DrgSystems::DRG_OPERATOR]))
         end
       end
 

data/lib/conceptql/vocabularies/entry.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "formats"
 require_relative "vocabulary_class_factory"
 
 module ConceptQL
@@ -13,7 +14,6 @@ module ConceptQL
         domain
         vocabulary_short_name
         vocabulary_long_name
-        format_regexp
       ].freeze
 
       METHODS.each do |meth|
@@ -23,6 +23,7 @@ module ConceptQL
       end
 
       METADATA_METHODS = METHODS + %i[
+        format_regexp
         preferred_name
         predominant_domains
       ]
@@ -54,6 +55,14 @@ module ConceptQL
         end
       end
 
+      # The anchored code format from vocabulary_formats, as a string, or nil
+      # when vocabulary_formats has no row for this vocabulary. A
+      # format_regexp column in a vocabularies CSV is ignored: vocabulary_formats
+      # is the only source (bead conceptql-bns).
+      def format_regexp
+        Formats.pattern(omopv5_id)
+      end
+
       def short_name
         vocabulary_short_name || vocabulary_long_name
       end
@@ -62,16 +71,19 @@ module ConceptQL
         vocabulary_long_name || vocabulary_short_name
       end
 
+      # A preferred_name in the hash wins: ms_drg and cms_drg select vocabulary
+      # DRG, but each is named after its own DRG system.
       def preferred_name
-        omopv5_id || id
+        hash[:preferred_name] || omopv5_id || id
       end
 
       def aliases
         @hash[:aliases_arr]
       end
 
+      # A vocabulary without a domain selects condition_occurrence records
       def predominant_domains
-        Array(domain || :condition_occurrence).flatten
+        Array(domain || "condition_occurrence").flatten
       end
 
       def from_lexicon?

data/lib/conceptql/vocabularies/formats.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "csv"
+require_relative "../paths"
+
+module ConceptQL
+  module Vocabularies
+    # The code formats that a Vocabulary operator checks its codes against.
+    # They come from config/vocabulary_formats.tsv, a pinned copy of the
+    # published table of the public repository
+    # outcomesinsights/vocabulary_formats (bead conceptql-bns, seed
+    # conceptql-ysx). config/vocabulary_formats.sha records the commit, and
+    # `just vocabulary-formats <ref>` refreshes both.
+    #
+    # Only the DEFAULT row of a vocabulary is used: the row whose vocabulary_id
+    # is the vocabulary's id. A variant row (ICD9CM.DOTLESS) is never used. The
+    # published patterns carry no anchors, so each one is anchored here as
+    # \A(?:pattern)\z. A vocabulary with no row has no format to check.
+    #
+    # A pattern is case-exact unless its row declares case-insensitivity with
+    # the scoped flag (?i:...) (vocabulary_formats seed vfm-8k1, seed
+    # conceptql-oji ruling "b"). Ruby honors the flag natively, so a pattern
+    # is compiled with no flag of its own: an ignore-case flag would override
+    # what the row says.
+    module Formats
+      DECLARES_CASE_INSENSITIVE = "(?i:"
+
+      class << self
+        # The anchored pattern of a vocabulary as a string, or nil. The
+        # vocabulary_id finds its row in any case.
+        def pattern(vocabulary_id)
+          patterns[vocabulary_id.to_s.downcase]
+        end
+
+        # The anchored pattern as a Regexp, or nil.
+        def regexp(vocabulary_id)
+          (str = pattern(vocabulary_id)) && Regexp.new(str)
+        end
+
+        # True when the vocabulary's row declares case-insensitivity with
+        # (?i:...). The lexicon then matches the vocabulary's codes without
+        # regard to case (LexiconStrategy#code_condition). A vocabulary with no
+        # row is case-exact.
+        def case_insensitive?(vocabulary_id)
+          pattern(vocabulary_id).to_s.include?(DECLARES_CASE_INSENSITIVE)
+        end
+
+        private
+
+        def patterns
+          @patterns ||= CSV.foreach(ConceptQL.vocabulary_formats_file_path, col_sep: "\t", headers: true)
+                           .reject { |row| row["vocabulary_id"].include?(".") }
+                           .to_h { |row| [ row["vocabulary_id"].downcase, "\\A(?:#{row.fetch("regexp")})\\z" ] }
+                           .freeze
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/vocabularies/vocabulary_class_factory.rb

@@ -3,6 +3,7 @@
 require_relative "../behaviors/drugish"
 require_relative "../behaviors/labish"
 require_relative "behaviors/gdmish"
+require_relative "drg_systems"
 require_relative "../operators/vocabulary"
 
 module ConceptQL
@@ -39,7 +40,7 @@ module ConceptQL
           preferred_name entry.preferred_name
           argument :codes, type: :codelist
           aliases entry.aliases
-          predominant_domains entry.predominant_domains
+          predominant_domains(*entry.predominant_domains)
           short_name entry.short_name
           long_name entry.long_name
           conceptql_spec_id "vocabulary"
@@ -62,6 +63,10 @@ module ConceptQL
 
           include ConceptQL::Behaviors::Drugish if entry.is_drugish?
 
+          include DrgSystems::OperatorBehavior if DrgSystems.system(entry.id)
+
+          deprecated DrgSystems::DRG_DEPRECATION if entry.id == DrgSystems::DRG_OPERATOR
+
           def vocab_entry
             self.class.entry
           end

data/lib/conceptql/window/table.rb

@@ -25,8 +25,11 @@ module ConceptQL
 
         rhs = query.db[get_table_window(query)]
 
+        # select_group replaces the select list, so rhs needs no
+        # remove_window_id. Its select_remove asked the database for the
+        # columns of the window table while the SQL was built (bead
+        # conceptql-6k8).
         unless opts[:precalculated_window_id]
-          rhs = remove_window_id(rhs)
           rhs = rhs
                 .select_group(:person_id, :start_date, :end_date)
                 .select_append do
@@ -37,7 +40,7 @@ module ConceptQL
 
         rhs = rhs.as(:r)
 
-        remove_window_id(query)
+        remove_window_id(query, op.dynamic_columns)
           .from_self(alias: :l)
           .join(rhs, expr)
           .select_all(:l)
@@ -52,12 +55,18 @@ module ConceptQL
         db[get_table_window(nil)].select(:person_id)
       end
 
-      def remove_window_id(ds)
+      # columns: the names of the columns of ds, when the caller knows them.
+      # For a ds without a select list, select_remove asks the database for
+      # them, and a table that ds reads may not exist while the SQL is built
+      # (bead conceptql-6k8).
+      def remove_window_id(ds, columns = nil)
         cols = ds.opts[:select]
         if cols && !cols.empty? && cols.any? { |c| !c.is_a?(Symbol) }
           # When select has non-symbol expressions (e.g. CAST...AS),
           # select_remove can't match them by symbol name, so filter manually
           ds.select(*cols.reject { |c| window_id_column?(c) })
+        elsif columns && cols.blank?
+          ds.select(*(columns - [ :window_id ]))
         else
           ds.select_remove(:window_id)
         end