conceptql 0.7.0 → 0.8.0

data/lib/conceptql/diagram_renderer/geometry.rb

@@ -228,8 +228,13 @@ module ConceptQL
         DOMAIN_LABEL_COLORS.fetch(stream_type, COUNT_LABEL[:fill])
       end
 
-      # formatCount: 1234567 becomes "1,234,567".
+      # formatCount: 1234567 becomes "1,234,567". A null count is NA: the
+      # data has no records of the operator's vocabulary (bead
+      # conceptql-d1bd). The JavaScript renderer has no such case, so the
+      # reference of a null count comes from this renderer.
       def format_count(n)
+        return "NA" if n.nil?
+
         JsFormat.string(n).gsub(/\B(?=(\d{3})+(?!\d))/, ",")
       end
 

data/lib/conceptql/lexicon/lexicon_strategy.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "sequel/adapters/mock"
 require_relative "../vocabularies/formats"
 
 module ConceptQL
@@ -140,22 +141,32 @@ module ConceptQL
       concepts(db, vocabulary_id, codes).select_map(%i[concept_code concept_text])
     end
 
-    # The distinct names of each code that has more than one name, as
-    # { downcased code => [name, ...] } with the names sorted. A vocabulary
-    # can reuse a code for a new meaning: DRG 410 meant chemotherapy until
-    # 2007-09-30 and a biliary tract procedure after that. OHDSI keeps each
-    # meaning as its own concept row, and the GDM vocabulary keeps both names.
-    # Names that differ only in case or spacing count as one name.
-    # A caller can answer "is the concept table missing?" with a block.
+    # The distinct names of each of the codes that has more than one name, as
+    # { code => [name, ...] } with the names sorted. A vocabulary can reuse a
+    # code for a new meaning: DRG 410 meant chemotherapy until 2007-09-30 and
+    # a biliary tract procedure after that. OHDSI keeps each meaning as its
+    # own concept row. Names that differ only in case or spacing count as one
+    # name.
+    #
+    # A stored code is one of the codes when it matches as code_condition
+    # matches: in its own case, unless the vocabulary declares (?i:...). So
+    # the Read codes 13g4.00 and 13G4.00 are two codes with one name each, not
+    # one code with two names (bead conceptql-qdn9). Each key is a code as the
+    # caller gave it. A caller can answer "is the concept table missing?" with
+    # a block.
     def code_meanings(data_db, vocabulary_id, codes)
       return {} if codes.blank? || db.nil? || db_is_mock?(data_db)
       return {} if block_given? ? yield : table_is_missing?(data_db)
 
-      concepts(db, vocabulary_id, codes)
-        .select_map(%i[concept_code concept_text])
-        .group_by { |code, _| code.downcase }
-        .transform_values { |rows| rows.map(&:last).compact.uniq { |name| name.downcase.split.join(" ") }.sort }
-        .select { |_, names| names.length > 1 }
+      key = case_insensitive?(vocabulary_id) ? :downcase.to_proc : :itself.to_proc
+      names = concepts(db, vocabulary_id, codes)
+              .select_map(%i[concept_code concept_text])
+              .group_by { |code, _| key.call(code) }
+              .transform_values { |rows| rows.map(&:last).compact.uniq { |name| name.downcase.split.join(" ") }.sort }
+              .select { |_, code_names| code_names.length > 1 }
+      Array(codes).each_with_object({}) do |code, meanings|
+        meanings[code] = names[key.call(code)] if names.key?(key.call(code))
+      end
     rescue Sequel::DatabaseError
       {}
     end

data/lib/conceptql/lexicon.rb

@@ -2,7 +2,6 @@
 
 require "forwardable"
 require_relative "code_case_mismatch_error"
-require_relative "lexicon/lexicon_gdm"
 require_relative "lexicon/lexicon_no_db"
 require_relative "lexicon/lexicon_ohdsi"
 
@@ -35,9 +34,6 @@ module ConceptQL
                    :vocabularies,
                    :vocabularies_query
 
-    # The key of each lexicon kind in NoVocabularyError::Search#missing_tables.
-    LEXICON_KINDS = { LexiconOhdsi => :ohdsi, LexiconGDM => :gdm }.freeze
-
     # The cached results keyed on [method, arguments], or nil when caching is
     # off. Its size is the number of distinct lookups.
     attr_reader :cache
@@ -48,8 +44,8 @@ module ConceptQL
     #
     # ConceptQL::Database raises it (through Lexicon.new) when its data
     # database is real, which is not nil and not a Sequel mock, and neither
-    # that database nor the LEXICON_URL database has all the tables of one
-    # lexicon kind. A database-less Database keeps LexiconNoDB.
+    # that database nor the LEXICON_URL database has all the OHDSI
+    # vocabulary tables. A database-less Database keeps LexiconNoDB.
     # MissingVocabularyError (Lexicon.for_dataset) is a subclass, so one
     # rescue covers both.
     class NoVocabularyError < StandardError
@@ -58,8 +54,8 @@ module ConceptQL
       # :dataset (Lexicon.for_dataset). +database+ names it without a user or
       # a password. +search_path+ is the search path that the unqualified
       # table names resolved through, or nil where the adapter has none.
-      # +missing_tables+ maps each lexicon kind (:ohdsi, :gdm) that was
-      # looked for to its tables that were not found.
+      # +missing_tables+ maps the lexicon kind :ohdsi to its tables that were
+      # not found.
       Search = Data.define(:role, :database, :search_path, :missing_tables)
 
       ROLES = {
@@ -78,8 +74,7 @@ module ConceptQL
       # The error for a ConceptQL::Database whose data database is real.
       # +database_type+ is the data database's, and chooses the advice.
       # +lexicon_url_searched+ is false when LEXICON_URL is not set.
-      # +only+ is the LEXICON_*_ONLY variable that narrowed the kinds, or nil.
-      def self.for_database(searches, database_type:, lexicon_url_searched:, only: nil)
+      def self.for_database(searches, database_type:, lexicon_url_searched:)
         data = searches.first
         lines = [
           "ConceptQL::Database found no vocabulary tables for its data database (#{data.database}).",
@@ -89,7 +84,6 @@ module ConceptQL
         ]
         lines.concat(searches.map { |search| "  - #{describe_search(search)}" })
         lines << "  - LEXICON_URL is not set, so no lexicon database was searched." unless lexicon_url_searched
-        lines << "#{only} is set, so only the #{only.delete_prefix("LEXICON_").delete_suffix("_ONLY")} tables were looked for." if only
         lines << "Fix it in one of these ways:"
         lines.concat(advice(database_type).map { |line| "  - #{line}" })
         new(lines.join("\n"), searches: searches)
@@ -110,7 +104,7 @@ module ConceptQL
         dataset = "Build the lexicon from the DuckDB dataset itself: " \
                   "ConceptQL::Database.new(db, lexicon: ConceptQL::Lexicon.for_dataset(db))."
         search_path = "Put the schema that holds the vocabulary tables on the data database's " \
-                      "search path (in the test data, ohdsi_vocabs or gdm_vocabs)."
+                      "search path (in the test data, ohdsi_vocabs)."
         lexicon_url = "Set LEXICON_URL to a database that has the vocabulary tables."
         case database_type
         when :duckdb then [ dataset, search_path, lexicon_url ]
@@ -221,9 +215,17 @@ module ConceptQL
 
     # The strategy's known_codes first asks whether the vocabulary is empty.
     # It asks through the block, so that question also reaches the cache.
-    def known_codes(data_db, vocabulary_id, codes, check_case: true)
-      known = lookup(:known_codes, [ data_db, vocabulary_id, codes ]) do
-        @strategy.known_codes(data_db, vocabulary_id, codes) { vocabulary_is_empty?(data_db, vocabulary_id) }
+    #
+    # An empty vocabulary knows every code, unless empty_means_known is
+    # false. Then it knows none. The code checks of the operators pass false
+    # and apply the 2020 rule (refs #185) through Database#check_codes,
+    # because the rule needs both lexicons: it applies only when the
+    # reference lexicon lacks the vocabulary too (bead conceptql-dufi).
+    def known_codes(data_db, vocabulary_id, codes, check_case: true, empty_means_known: true)
+      known = lookup(:known_codes, [ data_db, vocabulary_id, codes, empty_means_known ]) do
+        @strategy.known_codes(data_db, vocabulary_id, codes) do
+          empty_means_known && vocabulary_is_empty?(data_db, vocabulary_id)
+        end
       end
       check_case!(data_db, vocabulary_id, Array(codes) - known) if check_case
       known
@@ -300,19 +302,16 @@ module ConceptQL
     end
     private :lookup
 
-    # The first lexicon kind whose tables are all in the data db (dataset_db)
-    # or else the lexicon db. When neither has them, LexiconNoDB, but only
-    # for a database-less caller: a real data db raises NoVocabularyError.
+    # LexiconOhdsi on the data db (dataset_db) when it has all the OHDSI
+    # vocabulary tables, or else on the lexicon db. When neither has them,
+    # LexiconNoDB, but only for a database-less caller: a real data db raises
+    # NoVocabularyError.
     # A Sequel mock data db is not searched: it answers yes to every
     # table_exists?, so it would hide the lexicon db behind a vocabulary
     # that has no rows.
     def determine_strategy(lexicon_db, dataset_db)
       [ (dataset_db if real_database?(dataset_db)), lexicon_db ].compact.each do |db|
-        lexicon_classes.each do |klass|
-          next unless klass.db_has_all_vocabulary_tables?(db)
-
-          return (klass.new(db))
-        end
+        return LexiconOhdsi.new(db) if LexiconOhdsi.db_has_all_vocabulary_tables?(db)
       end
       raise no_vocabulary_error(lexicon_db, dataset_db) if real_database?(dataset_db)
 
@@ -328,15 +327,12 @@ module ConceptQL
     def no_vocabulary_error(lexicon_db, dataset_db)
       searches = [ search(:data_database, dataset_db) ]
       searches << search(:lexicon_url, lexicon_db) if lexicon_db
-      only = %w[LEXICON_OHDSI_ONLY LEXICON_GDM_ONLY].find { |name| ENV[name] }
       NoVocabularyError.for_database(searches, database_type: dataset_db.database_type.to_sym,
-                                               lexicon_url_searched: !lexicon_db.nil?, only: only)
+                                               lexicon_url_searched: !lexicon_db.nil?)
     end
 
     def search(role, db)
-      missing = lexicon_classes.to_h do |klass|
-        [ LEXICON_KINDS.fetch(klass), klass::VOCABULARY_TABLES.reject { |table| db.table_exists?(table) } ]
-      end
+      missing = { ohdsi: LexiconOhdsi::VOCABULARY_TABLES.reject { |table| db.table_exists?(table) } }
       NoVocabularyError::Search.new(role: role, database: describe_database(db),
                                     search_path: search_path_of(db), missing_tables: missing)
     end
@@ -368,12 +364,5 @@ module ConceptQL
       end
     end
     private :real_database?, :no_vocabulary_error, :search, :describe_database, :search_path_of
-
-    def lexicon_classes
-      classes = []
-      classes << LexiconOhdsi unless ENV["LEXICON_GDM_ONLY"]
-      classes << LexiconGDM unless ENV["LEXICON_OHDSI_ONLY"]
-      classes
-    end
   end
 end

data/lib/conceptql/nodifier.rb

@@ -56,6 +56,22 @@ module ConceptQL
       @operators ||= cdb.operators(@data_model)
     end
 
+    # The vocabulary ids of the data's clinical codes, as a Set, from the GDM
+    # table etl_vocabulary_ids (one row for each vocabulary in
+    # clinical_codes.clinical_code_vocabulary_id, written by the ETL). nil
+    # when the data has no such table, or when the table has no rows, which
+    # says nothing about the data (bead conceptql-d1bd). Only the
+    # vocabulary_id column is read, so the older 2-column table works too.
+    #
+    # Each Query has its own nodifier, so a Query reads the table once,
+    # whatever the number of its vocabulary operators.
+    def etl_vocabulary_ids(db)
+      return @etl_vocabulary_ids if defined?(@etl_vocabulary_ids)
+
+      ids = cdb.source_table_exists?(db, :etl_vocabulary_ids) ? db[:etl_vocabulary_ids].select_map(:vocabulary_id) : []
+      @etl_vocabulary_ids = ids.empty? ? nil : ids.to_set
+    end
+
     private
 
     def fetch_op(operator)

data/lib/conceptql/operators/co_reported.rb

@@ -31,6 +31,7 @@ module ConceptQL
       category "Combine Streams"
       default_query_columns
       validate_at_least_one_upstream
+      validate_at_least_two_upstreams
       validate_no_arguments
       require_column :context_id
 

data/lib/conceptql/operators/information_periods.rb

@@ -41,14 +41,18 @@ module ConceptQL
       private
 
       # A code that names no concept selects no period. A code is known only
-      # in its own case, as the lookup finds it.
+      # in its own case, as the lookup finds it. Database#check_codes sorts
+      # the codes that the dataset's lexicon does not know, with the 2020
+      # rule, as for a vocabulary operator (bead conceptql-dufi). The check
+      # reads only the lexicons, so it needs no data database.
       def additional_validation(db, opts = {})
-        return unless add_warnings?(db, opts)
+        return unless lexicon_checks?(opts)
 
         codes = dm.information_type_codes(arguments)
         return if codes.empty?
 
-        unknown = codes - dm.known_codes(db, dm.information_types_vocab, codes, check_case: false)
+        not_present, unknown = nodifier.cdb.check_codes(db, dm.information_types_vocab, codes)
+        add_warning(Vocabulary::NOT_PRESENT_IN_DATASET, *not_present) unless not_present.empty?
         add_warning(UNKNOWN_CODE, *unknown) unless unknown.empty?
       end
     end

data/lib/conceptql/operators/multiple_vocabularies.rb

@@ -120,34 +120,54 @@ module ConceptQL
       end
 
       # The per-code warnings a Vocabulary operator gives, from most to least
-      # specific (Operator::BAD_FORMATS, Operator::UNKNOWN_CODE). See
-      # #code_warnings.
-      CODE_WARNINGS = [ *BAD_FORMATS, UNKNOWN_CODE ].freeze
-
+      # specific (Operator::BAD_FORMATS, Vocabulary::NOT_PRESENT_IN_DATASET,
+      # Operator::UNKNOWN_CODE). This operator checks its codes itself (see
+      # #code_warnings), so it drops these warnings of its vocabulary
+      # operators.
+      CODE_WARNINGS = [ *BAD_FORMATS, Vocabulary::NOT_PRESENT_IN_DATASET, UNKNOWN_CODE ].freeze
+
+      # The operator checks its codes as one set across its vocabularies
+      # (bead conceptql-0h2n). When etl_vocabulary_ids lists none of its
+      # vocabularies, it gives the no-records warning and null counts, and
+      # its codes get no not-present warning, as a Vocabulary operator's do
+      # (bead conceptql-dufi). The other warnings of its vocabulary operators
+      # appear only when every one of them gives the warning with the same
+      # details (#aggregate_messages).
       def additional_validation(db, opts = {})
-        vocab_ops.each { |vo| vo.valid?(db, opts.merge(skip_merged_vocabulary_check: true)) }
+        @no_records_in_data = add_warnings?(db, opts) && vocab_ops.all? { |vo| vo.absent_from_data?(db) }
+        inner_opts = opts.merge(skip_merged_vocabulary_check: true)
+        vocab_ops.each { |vo| vo.valid?(db, inner_opts) }
 
         @errors += aggregate_messages(vocab_ops.map(&:errors))
         @warnings += aggregate_messages(vocab_ops.map { |vo| vo.warnings.reject { |w| CODE_WARNINGS.include?(w.first) } })
-        @warnings += code_warnings
-      end
-
-      # A code gets a warning only when every vocabulary rejects it, for any
-      # reason. It is an improperly formatted code only when every vocabulary
-      # says so. When at least one vocabulary accepts its format but does not
-      # know it, it is an unknown code. (12345 is badly formatted for HCPCS but
-      # a well-formed, unknown CPT4 code, so it is unknown.)
-      def code_warnings
-        verdicts = vocab_ops.map do |vo|
-          vo.warnings.each_with_object({}) do |(message, *codes), verdict|
-            codes.each { |code| verdict[code] ||= message } if CODE_WARNINGS.include?(message)
-          end
-        end
+        add_warning(Vocabulary::NO_RECORDS_IN_DATA) if @no_records_in_data
+        @warnings += code_warnings(db, opts)
+      end
 
-        rejected = arguments.uniq.select { |code| verdicts.all? { |verdict| verdict.key?(code) } }
-        bad_format, unknown = rejected.partition { |code| verdicts.all? { |verdict| BAD_FORMATS.include?(verdict[code]) } }
+      def no_records_in_data?(db, opts = {})
+        validate(db, opts)
+        @no_records_in_data || false
+      end
 
-        [ [ BAD_FORMAT, *bad_format ], [ UNKNOWN_CODE, *unknown ] ].reject { |warning| warning.length == 1 }
+      # A code is improperly formatted when every vocabulary rejects its
+      # format. (12345 is badly formatted for HCPCS but has the format of
+      # CPT4, so it is checked.) Database#check_codes sorts the other codes
+      # across all of the vocabularies: a code is known when the Database's
+      # lexicon has it in any of them, and the 2020 rule applies only when no
+      # lexicon has a concept of any of them. With no records of the
+      # vocabularies in the data, the not-present warning is not given. Each
+      # warning lists every code.
+      def code_warnings(db, opts = {})
+        return [] if arguments.include?("*")
+
+        codes = arguments.uniq
+        bad_format = codes.select { |code| vocab_ops.all? { |vo| vo.bad_arguments.include?(code) } }
+        checked = codes - bad_format
+        not_present, unknown = lexicon_checks?(opts) ? nodifier.cdb.check_codes(db, vocab_ops.map(&:vocabulary_id), checked) : [ [], [] ]
+        not_present = [] if @no_records_in_data
+
+        [ [ BAD_FORMAT, *bad_format ], [ Vocabulary::NOT_PRESENT_IN_DATASET, *not_present ], [ UNKNOWN_CODE, *unknown ] ]
+          .reject { |warning| warning.length == 1 }
       end
 
       def aggregate_messages(message_sets)

data/lib/conceptql/operators/operator.rb

@@ -48,6 +48,7 @@ module ConceptQL
       NDC_BAD_FORMAT = "#{BAD_FORMAT} (claims use 11-digit package NDCs, so it matches no rows)"
       BAD_FORMATS = [ BAD_FORMAT, NDC_BAD_FORMAT ].freeze
       UNKNOWN_CODE = "unknown code(s)"
+      ONE_UPSTREAM = "has only one upstream"
 
       option :label, type: :string,
                      meaning: "A short description of the operator's records. The operator writes it into the label column of each of its records, and a recall operator selects those records by it. Each label in a statement must be different."
@@ -80,6 +81,7 @@ module ConceptQL
           no_upstreams
           one_upstream
           at_least_one_upstream
+          at_least_two_upstreams
           at_most_one_upstream
           no_arguments
           one_argument
@@ -171,7 +173,6 @@ module ConceptQL
       def annotate(db, opts = {})
         return @annotate if defined?(@annotate)
 
-        scope_key = options[:id] || op_name
         annotation = {}
         counts = (annotation[:counts] ||= {})
         metadata = { annotation: annotation }
@@ -181,24 +182,23 @@ module ConceptQL
         end
         res = [ op_name, *annotate_values(db, opts) ]
 
+        # An operator of a vocabulary that the data does not contain has null
+        # counts, and so does an operator whose inputs all have null counts.
+        # A 0 would say that no patient has the codes (bead conceptql-d1bd).
+        no_records = no_records_in_data?(db, opts)
         if countable?(db, opts)
-          counts.merge!(scope.one_query_counts&.for(self) || domain_counts(db))
+          counts.merge!(scope.one_query_counts&.for(self) || domain_counts(db)) unless no_records
         elsif !errors.empty?
           annotation[:errors] = errors
-          scope.add_errors(scope_key, errors)
         end
 
-        domains(db).each do |domain|
-          cur_counts = counts[domain] ||= { rows: 0, n: 0 }
-          scope.add_counts(scope_key, domain, cur_counts)
-        end
+        domains(db).each { |domain| counts[domain] ||= no_records ? { rows: nil, n: nil } : { rows: 0, n: 0 } }
         # The database returns its groups in no fixed order, and the zero
         # counts above come last. Sort them, so each run gives the same order.
         annotation[:counts] = counts.sort.to_h
 
         if defined?(@warnings) && !warnings.empty?
           annotation[:warnings] = warnings
-          scope.add_warnings(scope_key, warnings.dup)
         end
 
         if res.last.is_a?(Hash)
@@ -340,6 +340,15 @@ module ConceptQL
         upstreams_valid?(db, opts) && scope.valid? && include_counts?(db, opts)
       end
 
+      # Whether the data has no records for this operator to count, so its
+      # counts are null. A Vocabulary operator decides for itself. Any other
+      # operator has none when every one of its inputs has none. An operator
+      # with one real input keeps its real counts (bead conceptql-d1bd).
+      def no_records_in_data?(db, opts = {})
+        inputs = all_upstreams.compact
+        !inputs.empty? && inputs.all? { |input| input.no_records_in_data?(db, opts) }
+      end
+
       # Rows and distinct people of each criterion_domain in this operator's
       # output, with a query for this operator alone.
       def domain_counts(db)
@@ -564,6 +573,14 @@ module ConceptQL
         add_error("has no upstream") if @upstreams.empty?
       end
 
+      # A warning, not an error: an operator that combines its upstreams
+      # passes the records of one upstream through unchanged, which is almost
+      # never what the author meant. Zero upstreams is the error of
+      # validate_at_least_one_upstream, so declare both.
+      def validate_at_least_two_upstreams
+        add_warning(ONE_UPSTREAM) if @upstreams.length == 1
+      end
+
       def validate_no_arguments
         add_error("has arguments", @arguments) unless @arguments.empty?
       end
@@ -617,6 +634,10 @@ module ConceptQL
         end
       end
 
+      # MultipleVocabularies reads the bad codes of each of its vocabulary
+      # operators.
+      protected :bad_arguments
+
       # Only Vocabulary operators carry a code format (Vocabulary#code_regexp)
       def code_regexp
         nil
@@ -641,6 +662,15 @@ module ConceptQL
         @errors.empty? && !no_db?(db, opts)
       end
 
+      # Whether to run the checks that read only the lexicon, such as the
+      # unknown-code check. They need no data database, so they also run for
+      # a Database whose data database is nil or a Sequel mock but that has a
+      # lexicon (the Jigsaw editor, bead conceptql-0ubo). skip_db turns them
+      # off, as it turns off add_warnings?.
+      def lexicon_checks?(opts = {})
+        @errors.empty? && dm.lexicon.strategy != :no_db && !skip_db?(opts)
+      end
+
       def add_error(*args)
         errors << args
       end

data/lib/conceptql/operators/place_of_service_filter.rb

@@ -16,8 +16,8 @@ module ConceptQL
     #
     # Multiple places of service can be specified at once. Each code becomes
     # a concept ID through the lexicon, so a code that names no concept can
-    # match no record. Such a code is an error, as an unrecognized
-    # provenance keyword is.
+    # match no record. Such a code gets a warning, as an unknown provenance
+    # keyword does.
     #
     # The code "*" keeps a record whose place of service is any concept of
     # the vocabulary, as provenance's "*" keeps any provenance concept. A
@@ -38,7 +38,7 @@ module ConceptQL
       argument :places_of_service, type: :codelist, vocab: VOCABULARY_ID,
                                    meaning: "One or more CMS Place of Service codes, such as 21 (inpatient hospital) or 22 (outpatient hospital). " \
                                             "The code * matches a record whose place of service is any code of that vocabulary. " \
-                                            "An unknown code makes the statement invalid."
+                                            "An unknown code gets a warning and matches no record."
 
       # The test data has no place of service, so the example returns no rows
       example "Diabetes diagnoses (ICD-9 250.00) with the place of service outpatient hospital (22)",
@@ -60,22 +60,34 @@ module ConceptQL
       private
 
       # With no codes, the lookup returns every concept of the vocabulary.
-      # The codes come from the statement, whose validation reports an
+      # The codes come from the statement, whose validation warns about an
       # unknown code, so the lookup does not raise CodeCaseMismatchError.
       def place_of_service_concept_ids(db)
         codes = arguments.map(&:to_s)
         dm.concept_ids(db, VOCABULARY_ID, codes.include?(ANY) ? [] : codes, check_case: false)
       end
 
-      # With no lexicon there are no codes to check against, and every code
-      # would be reported. Provenance skips its check in the same situation.
-      # A code is known only in its own case, as the lookup finds it.
-      def additional_validation(db, _opts = {})
-        return if dm.lexicon.strategy == :no_db
+      # A code that the dataset's lexicon does not know is a warning, as an
+      # unknown code of a vocabulary operator is: "code(s) not present in
+      # this dataset" when the reference lexicon knows it, and else "unknown
+      # code(s)" (bead conceptql-amuu). A code is known only in its own case,
+      # as the lookup finds it. The check reads only the lexicons, so it
+      # needs no data database.
+      def additional_validation(db, opts = {})
+        return unless lexicon_checks?(opts)
 
-        known = dm.concept_codes(db, VOCABULARY_ID)
-        unknown = arguments.reject { |code| code.to_s == ANY || known.include?(code.to_s) }
-        add_error(UNKNOWN_CODE, *unknown) unless unknown.empty?
+        codes = arguments.map(&:to_s).uniq - [ ANY ]
+        return if codes.empty?
+
+        not_present, unknown = nodifier.cdb.check_codes(db, VOCABULARY_ID, codes)
+        add_warning(Vocabulary::NOT_PRESENT_IN_DATASET, *as_written(not_present)) unless not_present.empty?
+        add_warning(UNKNOWN_CODE, *as_written(unknown)) unless unknown.empty?
+      end
+
+      # The arguments whose text is one of the codes, as written and in the
+      # order written. count_person_with_co_reported writes a code as a number.
+      def as_written(codes)
+        arguments.select { |code| codes.include?(code.to_s) }.uniq
       end
     end
   end

data/lib/conceptql/operators/provenance.rb

@@ -33,7 +33,7 @@ module ConceptQL
                                            "A keyword also matches the provenances under it, so inpatient includes snf. " \
                                            "A keyword of the form file:code, such as inpatient:primary, needs both the file and the code provenance to match. " \
                                            "The keyword * matches a record that has any provenance. " \
-                                           "A statement with no keyword, or with an unknown keyword, is invalid."
+                                           "A statement with no keyword is invalid. An unknown keyword gets a warning and matches no record."
 
       example "Diabetes diagnoses (ICD-9 250.00) from inpatient files",
               [ "provenance", "inpatient", [ "icd9", "250.00" ] ]
@@ -55,18 +55,18 @@ module ConceptQL
 
       private
 
-      def additional_validation(db, _opts = {})
+      # A keyword that the dataset's lexicon does not know is a warning, as
+      # an unknown code of a vocabulary operator is: "code(s) not present in
+      # this dataset" when the reference lexicon knows it, and else "unknown
+      # code(s)" (bead conceptql-amuu). Such a keyword selects no record. The
+      # check reads only the lexicons, so it needs no data database.
+      def additional_validation(db, opts = {})
         return add_error('no provenance keywords; use "*" for any provenance') if arguments.empty?
+        return unless lexicon_checks?(opts)
 
-        # With no lexicon there are no provenance concepts to check against,
-        # and every keyword would be reported as unrecognized. Vocabulary
-        # operators skip their unknown-code check in the same situation.
-        return if dm.lexicon.strategy == :no_db
-
-        bad_keywords = find_bad_keywords(db, arguments)
-        return unless bad_keywords.present?
-
-        add_error("unrecognized keywords", *bad_keywords.uniq)
+        not_present, unknown = check_keywords(db, arguments)
+        add_warning(Vocabulary::NOT_PRESENT_IN_DATASET, *not_present) unless not_present.empty?
+        add_warning(UNKNOWN_CODE, *unknown) unless unknown.empty?
       end
     end
   end

data/lib/conceptql/operators/recall.rb

@@ -79,6 +79,11 @@ module ConceptQL
         nodifier.scope.fetch_operator(source)
       end
 
+      # The input of a recall is the labeled operator.
+      def no_records_in_data?(db, opts = {})
+        valid?(db, opts) && original.no_records_in_data?(db, opts)
+      end
+
       def code_list(db)
         original.code_list(db)
       end

data/lib/conceptql/operators/union.rb

@@ -10,12 +10,22 @@ module ConceptQL
       class Combiner
         attr_reader :combinables
 
+        # The vocabulary leaves that one scan can select, in groups: the
+        # leaves of one domain and one column family. The scan takes its
+        # columns from its first leaf, so a leaf with the lab columns (read,
+        # whose domain is condition_occurrence) gets its own scan, and an icd9
+        # leaf beside it keeps the default columns (bead conceptql-qdn9).
+        def self.scans(combinables)
+          combinables.group_by { |combo| [ combo.domain, combo.column_family ] }.values
+        end
+
         def initialize(combinables)
           @combinables = combinables
         end
 
         def queries(db)
-          combinables.group_by(&:domain).map do |domain, combos|
+          self.class.scans(combinables).map do |combos|
+            domain = combos.first.domain
             include_uuid = combos.any? { |c| c.options[:uuid] }
             criteria = combos.map do |combo|
               combo.filter_clause(db)
@@ -71,6 +81,7 @@ module ConceptQL
       category "Combine Streams"
       default_query_columns
       validate_at_least_one_upstream
+      validate_at_least_two_upstreams
       validate_no_arguments
 
       def query(db)
@@ -112,7 +123,7 @@ module ConceptQL
       def disjoint_upstreams?
         combinables, individuals = upstreams.partition { |upstream| upstream.is_a?(Vocabulary) }
         queries = individuals.map { |upstream| branches_of(upstream) }
-        queries += combinables.group_by(&:domain).values.map { |combos| combos.flat_map { |c| branches_of(c) } }
+        queries += Combiner.scans(combinables).map { |combos| combos.flat_map { |c| branches_of(c) } }
         return false unless queries.all?
 
         queries.combination(2).all? do |one, other|