conceptql 0.4.1 → 0.6.0

data/lib/conceptql/lexicon/lexicon_strategy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../vocabularies/formats"
+
 module ConceptQL
   class LexiconStrategy
     attr_reader :db
@@ -8,9 +10,9 @@ module ConceptQL
       @db = db
     end
 
-    # A lookup is always scoped to a vocabulary. `vocabulary_id` is polymorphic:
-    # a String, an Integer (ReadOmop reports 17), or an Array of vocabulary names
-    # (the provenance-type pairs), which Sequel turns into an IN clause.
+    # A lookup is always scoped to a vocabulary. `vocabulary_id` is a String or
+    # an Array of vocabulary names (the provenance-type pairs), which Sequel turns
+    # into an IN clause.
     #
     # There is deliberately no wildcard. `'*'` is meaningful elsewhere in the
     # operator family, but only in `arguments` — Vocabulary#select_all? reads it
@@ -28,11 +30,35 @@ module ConceptQL
       ds = concepts_table(data_db)
 
       ds = ds.where(vocabulary_id: vocabulary_id)
-      ds = ds.where(Sequel.function(:lower, :concept_code) => Array(codes).map(&:downcase)) unless codes.blank?
+      ds = ds.where(code_condition(vocabulary_id, codes)) unless codes.blank?
 
       ds
     end
 
+    # The condition that matches concept_code with codes. It compares case
+    # exactly, unless the vocabulary's vocabulary_formats row declares
+    # case-insensitivity with (?i:...) (seed conceptql-oji ruling "b"). Then it
+    # compares lower(concept_code) with the codes in lower case. One lookup
+    # over several vocabularies has one comparison, so they must agree.
+    def code_condition(vocabulary_id, codes)
+      codes = Array(codes)
+      return { concept_code: codes } unless case_insensitive?(vocabulary_id)
+
+      { Sequel.function(:lower, :concept_code) => codes.map(&:downcase) }
+    end
+
+    def case_insensitive?(vocabulary_id)
+      ids = Array(vocabulary_id)
+      declared = ids.select { |id| Vocabularies::Formats.case_insensitive?(id) }
+      return false if declared.empty?
+      return true if declared.size == ids.size
+
+      raise ArgumentError,
+            "one lookup over the vocabularies #{ids.inspect} cannot compare case: " \
+            "#{declared.inspect} declare case-insensitivity with (?i:...) in vocabulary_formats, " \
+            "and #{(ids - declared).inspect} do not"
+    end
+
     def concepts_by_name(data_db, names = [])
       ds = concepts_table(data_db)
 
@@ -98,6 +124,34 @@ module ConceptQL
       concepts(db, vocabulary_id, codes).select_map(%i[concept_code concept_text])
     end
 
+    # The distinct names of each code that has more than one name, as
+    # { downcased code => [name, ...] } with the names sorted. A vocabulary
+    # can reuse a code for a new meaning: DRG 410 meant chemotherapy until
+    # 2007-09-30 and a biliary tract procedure after that. OHDSI keeps each
+    # meaning as its own concept row, and the GDM vocabulary keeps both names.
+    # Names that differ only in case or spacing count as one name.
+    # A caller can answer "is the concept table missing?" with a block.
+    def code_meanings(data_db, vocabulary_id, codes)
+      return {} if codes.blank? || db.nil? || db_is_mock?(data_db)
+      return {} if block_given? ? yield : table_is_missing?(data_db)
+
+      concepts(db, vocabulary_id, codes)
+        .select_map(%i[concept_code concept_text])
+        .group_by { |code, _| code.downcase }
+        .transform_values { |rows| rows.map(&:last).compact.uniq { |name| name.downcase.split.join(" ") }.sort }
+        .select { |_, names| names.length > 1 }
+    rescue Sequel::DatabaseError
+      {}
+    end
+
+    # The [code, name] pairs of the codes whose concept is in one concept
+    # class, or nil when the lexicon keeps no concept classes. Only the OHDSI
+    # vocabulary keeps them (LexiconOhdsi). ms_drg and cms_drg name a code by
+    # its meaning in their own DRG system with it (DrgSystems).
+    def concepts_to_codes_in_class(_data_db, _vocabulary_id, _concept_class_id, _codes)
+      nil
+    end
+
     def vocabulary_is_empty?(data_db, vocabulary_id)
       concepts_table(data_db).where(vocabulary_id: vocabulary_id).count.zero?
     end

data/lib/conceptql/lexicon.rb

@@ -14,10 +14,12 @@ module ConceptQL
     # cache never outlives one export (seed conceptql-a5f). Methods that return
     # datasets are not cached: their query runs later, in the caller.
     CACHED_LOOKUPS = %i[
+      code_meanings
       concept_codes
       concept_ids
       concept_ids_by_name
       concepts_to_codes
+      concepts_to_codes_in_class
       descendants_of
       known_codes
       related_concept_ids
@@ -34,16 +36,158 @@ module ConceptQL
                    :vocabularies,
                    :vocabularies_query
 
+    # The key of each lexicon kind in NoVocabularyError::Search#missing_tables.
+    LEXICON_KINDS = { LexiconOhdsi => :ohdsi, LexiconGDM => :gdm }.freeze
+
     # The cached results keyed on [method, arguments], or nil when caching is
     # off. Its size is the number of distinct lookups.
     attr_reader :cache
 
+    # No database that ConceptQL searched has the vocabulary tables. Without
+    # them every code compiles to 1 = 0 and the unknown-code checks turn off,
+    # so a query on real data would return no rows without a word.
+    #
+    # ConceptQL::Database raises it (through Lexicon.new) when its data
+    # database is real, which is not nil and not a Sequel mock, and neither
+    # that database nor the LEXICON_URL database has all the tables of one
+    # lexicon kind. A database-less Database keeps LexiconNoDB.
+    # MissingVocabularyError (Lexicon.for_dataset) is a subclass, so one
+    # rescue covers both.
+    class NoVocabularyError < StandardError
+      # One database that was searched. +role+ is :data_database (the
+      # Database's own db), :lexicon_url (the LEXICON_URL database) or
+      # :dataset (Lexicon.for_dataset). +database+ names it without a user or
+      # a password. +search_path+ is the search path that the unqualified
+      # table names resolved through, or nil where the adapter has none.
+      # +missing_tables+ maps each lexicon kind (:ohdsi, :gdm) that was
+      # looked for to its tables that were not found.
+      Search = Data.define(:role, :database, :search_path, :missing_tables)
+
+      ROLES = {
+        data_database: "the data database",
+        lexicon_url: "the LEXICON_URL database",
+        dataset: "the DuckDB dataset"
+      }.freeze
+
+      attr_reader :searches
+
+      def initialize(message = nil, searches: [])
+        @searches = searches
+        super(message)
+      end
+
+      # The error for a ConceptQL::Database whose data database is real.
+      # +database_type+ is the data database's, and chooses the advice.
+      # +lexicon_url_searched+ is false when LEXICON_URL is not set.
+      # +only+ is the LEXICON_*_ONLY variable that narrowed the kinds, or nil.
+      def self.for_database(searches, database_type:, lexicon_url_searched:, only: nil)
+        data = searches.first
+        lines = [
+          "ConceptQL::Database found no vocabulary tables for its data database (#{data.database}).",
+          "Without them every code would match no rows and unknown codes would not be reported, " \
+          "so the Database refuses to start.",
+          "It searched:"
+        ]
+        lines.concat(searches.map { |search| "  - #{describe_search(search)}" })
+        lines << "  - LEXICON_URL is not set, so no lexicon database was searched." unless lexicon_url_searched
+        lines << "#{only} is set, so only the #{only.delete_prefix("LEXICON_").delete_suffix("_ONLY")} tables were looked for." if only
+        lines << "Fix it in one of these ways:"
+        lines.concat(advice(database_type).map { |line| "  - #{line}" })
+        new(lines.join("\n"), searches: searches)
+      end
+
+      def self.describe_search(search)
+        where = ROLES.fetch(search.role)
+        where += " (#{search.database})" unless search.role == :data_database
+        where += ", through the search path #{search.search_path.inspect}" if search.search_path
+        missing = search.missing_tables.map do |kind, tables|
+          "#{kind.to_s.upcase} tables #{tables.join(", ")} are missing"
+        end
+        "#{where}: #{missing.join("; ")}."
+      end
+      private_class_method :describe_search
+
+      def self.advice(database_type)
+        dataset = "Build the lexicon from the DuckDB dataset itself: " \
+                  "ConceptQL::Database.new(db, lexicon: ConceptQL::Lexicon.for_dataset(db))."
+        search_path = "Put the schema that holds the vocabulary tables on the data database's " \
+                      "search path (in the test data, ohdsi_vocabs or gdm_vocabs)."
+        lexicon_url = "Set LEXICON_URL to a database that has the vocabulary tables."
+        case database_type
+        when :duckdb then [ dataset, search_path, lexicon_url ]
+        when :postgres then [ search_path, lexicon_url ]
+        else [ lexicon_url ]
+        end
+      end
+      private_class_method :advice
+    end
+
+    # A DuckDB dataset lacks one or more of the OHDSI vocabulary tables, so
+    # Lexicon.for_dataset cannot make the dataset its own lexicon.
+    class MissingVocabularyError < NoVocabularyError
+      attr_reader :missing_tables, :schemas
+
+      # +missing_tables+ are the vocabulary tables that were not found.
+      # +schemas+ are the schemas that were searched for them, each as
+      # "catalog.schema", and +search_path+ the DuckDB search path that the
+      # unqualified table names resolved through.
+      def initialize(missing_tables, schemas:, search_path:)
+        @missing_tables = missing_tables
+        @schemas = schemas
+        search = Search.new(role: :dataset, database: "schemas #{schemas.join(", ")}",
+                            search_path: search_path, missing_tables: { ohdsi: missing_tables })
+        super("The DuckDB dataset (schemas #{schemas.map(&:inspect).join(", ")}) " \
+              "has no vocabulary table #{missing_tables.join(", ")}. " \
+              "A DuckDB dataset is its own lexicon: ConceptQL reads " \
+              "#{LexiconOhdsi::VOCABULARY_TABLES.join(", ")} only from it, and never from LEXICON_URL. " \
+              "The tables were looked up without a schema, through the search path #{search_path.inspect}.",
+              searches: [ search ])
+      end
+    end
+
+    # The lexicon of a DuckDB dataset: the OHDSI vocabulary tables of db
+    # itself, found without a schema through db's search path. LEXICON_URL is
+    # never read. The Lexicon caches its lookups for its own life. Pass it to
+    # ConceptQL::Database.new(db, lexicon: lexicon), without cache_lookups.
+    #
+    # Raises MissingVocabularyError when db lacks any of the tables, because a
+    # DuckDB dataset never falls back to another lexicon.
+    #
+    # Returns nil for a Postgres or Spark db. The caller then builds its
+    # Database without lexicon:, and the Database chooses one as before.
+    def self.for_dataset(db)
+      return unless db.database_type.to_sym == :duckdb
+
+      missing = LexiconOhdsi::VOCABULARY_TABLES.reject { |table| db.table_exists?(table) }
+      unless missing.empty?
+        search_path = db.get(Sequel.function(:current_setting, "search_path"))
+        raise MissingVocabularyError.new(missing, schemas: searched_schemas(db, search_path), search_path: search_path)
+      end
+
+      new(nil, db, strategy: LexiconOhdsi.new(db), cache: true)
+    end
+
+    # The schemas, as "catalog.schema", where DuckDB looks for a table named
+    # without a schema. DuckDB reports a set search path in that form. The
+    # consumers attach the dataset and search, for example,
+    # "memory.main,dataset_<schema>.<schema>" (sequelizer 0.5 with_dataset),
+    # so current_schema() alone ("main") would name the wrong schema. With no
+    # search path set, DuckDB searches the current schema (and its own system
+    # schemas, which hold no dataset tables).
+    def self.searched_schemas(db, search_path)
+      schemas = search_path.to_s.split(",").map(&:strip).reject(&:empty?)
+      return schemas unless schemas.empty?
+
+      [ "#{db.get(Sequel.function(:current_database))}.#{db.get(Sequel.function(:current_schema))}" ]
+    end
+    private_class_method :searched_schemas
+
     def initialize(lexicon_db, dataset_db = nil, strategy: nil, cache: false)
       @strategy = strategy || determine_strategy(lexicon_db, dataset_db)
       @cache = {} if cache
     end
 
-    (CACHED_LOOKUPS - %i[concepts_to_codes known_codes table_is_missing?]).each do |name|
+    (CACHED_LOOKUPS - %i[code_meanings concepts_to_codes concepts_to_codes_in_class known_codes table_is_missing?]).each do |name|
       define_method(name) do |*args|
         lookup(name, args) { @strategy.public_send(name, *args) }
       end
@@ -65,6 +209,20 @@ module ConceptQL
       end
     end
 
+    # Like concepts_to_codes, it asks about the concept table through the block.
+    def concepts_to_codes_in_class(data_db, vocabulary_id, concept_class_id, codes)
+      lookup(:concepts_to_codes_in_class, [ data_db, vocabulary_id, concept_class_id, codes ]) do
+        @strategy.concepts_to_codes_in_class(data_db, vocabulary_id, concept_class_id, codes) { table_is_missing?(data_db) }
+      end
+    end
+
+    # Like concepts_to_codes, it asks about the concept table through the block.
+    def code_meanings(data_db, vocabulary_id, codes)
+      lookup(:code_meanings, [ data_db, vocabulary_id, codes ]) do
+        @strategy.code_meanings(data_db, vocabulary_id, codes) { table_is_missing?(data_db) }
+      end
+    end
+
     # Only "the table is there" is kept. A missing table (or a database error,
     # which Sequel's table_exists? turns into "missing") is checked again on
     # the next call, so one transient error does not last the whole export.
@@ -90,17 +248,75 @@ module ConceptQL
     end
     private :lookup
 
+    # The first lexicon kind whose tables are all in the data db (dataset_db)
+    # or else the lexicon db. When neither has them, LexiconNoDB, but only
+    # for a database-less caller: a real data db raises NoVocabularyError.
+    # A Sequel mock data db is not searched: it answers yes to every
+    # table_exists?, so it would hide the lexicon db behind a vocabulary
+    # that has no rows.
     def determine_strategy(lexicon_db, dataset_db)
-      [ dataset_db, lexicon_db ].compact.each do |db|
+      [ (dataset_db if real_database?(dataset_db)), lexicon_db ].compact.each do |db|
         lexicon_classes.each do |klass|
           next unless klass.db_has_all_vocabulary_tables?(db)
 
           return (klass.new(db))
         end
       end
+      raise no_vocabulary_error(lexicon_db, dataset_db) if real_database?(dataset_db)
+
       LexiconNoDB.new(Sequel.mock(host: :postgres))
     end
 
+    # Not nil and not a Sequel mock: the diagram renderer and SQL rendering
+    # use those, and they have no vocabulary on purpose.
+    def real_database?(db)
+      !db.nil? && db.adapter_scheme != :mock
+    end
+
+    def no_vocabulary_error(lexicon_db, dataset_db)
+      searches = [ search(:data_database, dataset_db) ]
+      searches << search(:lexicon_url, lexicon_db) if lexicon_db
+      only = %w[LEXICON_OHDSI_ONLY LEXICON_GDM_ONLY].find { |name| ENV[name] }
+      NoVocabularyError.for_database(searches, database_type: dataset_db.database_type.to_sym,
+                                               lexicon_url_searched: !lexicon_db.nil?, only: only)
+    end
+
+    def search(role, db)
+      missing = lexicon_classes.to_h do |klass|
+        [ LEXICON_KINDS.fetch(klass), klass::VOCABULARY_TABLES.reject { |table| db.table_exists?(table) } ]
+      end
+      NoVocabularyError::Search.new(role: role, database: describe_database(db),
+                                    search_path: search_path_of(db), missing_tables: missing)
+    end
+
+    # The database's type, name and host, never its user or password. A
+    # DuckDB database is named by its file, which DuckDB reports itself:
+    # Sequel's URL parsing drops the leading "/" of a duckdb:///path.
+    def describe_database(db)
+      name = db.opts[:database]
+      if db.database_type.to_sym == :duckdb
+        name = db.from(Sequel.function(:duckdb_databases))
+                 .where(database_name: Sequel.function(:current_database))
+                 .get(:path) || ":memory:"
+      end
+      host = db.opts[:host].to_s
+      [ "#{db.database_type} database",
+        (name.to_s.inspect if name),
+        ("on host #{host.inspect}" unless host.empty?) ].compact.join(" ")
+    end
+
+    # The search path that a table named without a schema resolves through:
+    # Postgres's search_path setting, DuckDB's searched schemas as
+    # "catalog.schema", or nil for an adapter without one.
+    def search_path_of(db)
+      case db.database_type.to_sym
+      when :postgres then db.get(Sequel.function(:current_setting, "search_path"))
+      when :duckdb
+        self.class.send(:searched_schemas, db, db.get(Sequel.function(:current_setting, "search_path"))).join(",")
+      end
+    end
+    private :real_database?, :no_vocabulary_error, :search, :describe_database, :search_path_of
+
     def lexicon_classes
       classes = []
       classes << LexiconOhdsi unless ENV["LEXICON_GDM_ONLY"]

data/lib/conceptql/nodifier.rb

@@ -20,7 +20,13 @@ module ConceptQL
       (opts[:data_model] || cdb.opts[:data_model] || ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym
     end
 
-    def create(operator, *values)
+    # An empty statement ([], as a whole statement, an operand, or the stored
+    # statement of an algorithm) has no operator. It becomes an Invalid
+    # operator with the error "empty statement", so validation reports it.
+    def create(*statement)
+      return Operators::Invalid.new(self, "invalid", errors: [ [ "empty statement" ] ]) if statement.empty?
+
+      operator, *values = statement
       operator = operator.to_s.downcase
       if operator == "algorithm"
         statement, = algorithm_fetcher.call(values.first)

data/lib/conceptql/one_query_counts.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  # Counts the rows and the distinct people of every operator in a statement
+  # with one SQL statement, for Operator#annotate. The SQL statement is the
+  # root query's CTEs plus a UNION ALL of one count for each operator's CTE.
+  #
+  # This needs MATERIALIZED CTEs, so that each CTE runs once. Spark does not
+  # honor MATERIALIZED, so on Spark each operator counts its rows with its own
+  # query (Operator#domain_counts). That per-operator path is also used when
+  # the statement has an operator that #annotate does not count (an invalid
+  # operator, or no database), with temp tables instead of CTEs, and with the
+  # Database's share_utilization option, whose tables are not CTEs.
+  #
+  # Nothing runs until the first operator asks for its counts.
+  class OneQueryCounts
+    OPERATOR_COLUMN = :conceptql_operator
+
+    def initialize(root, db, opts)
+      @root = root
+      @db = db
+      @opts = opts
+    end
+
+    # The counts of op as { criterion_domain => { rows:, n: } }, without the
+    # domains that have no rows. nil if this statement cannot be counted with
+    # one SQL statement.
+    def for(op)
+      counts&.fetch(op)
+    end
+
+    # The SQL statement that counts every operator, or nil.
+    def dataset
+      return @dataset if defined?(@dataset)
+
+      @dataset = build_dataset
+    end
+
+    private
+
+    attr_reader :root, :db, :opts
+
+    def scope
+      root.scope
+    end
+
+    def counts
+      return @counts if defined?(@counts)
+
+      @counts = dataset && read_counts
+    end
+
+    def read_counts
+      by_index = Array.new(@operators.length) { {} }
+      dataset.each do |h|
+        by_index[h[OPERATOR_COLUMN]][h[:criterion_domain].to_sym] = { rows: h[:rows], n: h[:n] }
+      end
+      @operators.zip(by_index).to_h.compare_by_identity
+    end
+
+    def build_dataset
+      return unless root.rdbms.supports_materialized?
+      return if scope.force_temp_tables? || ConceptQL.avoid_ctes?
+
+      @operators = operators
+      return unless @operators.all? { |op| op.countable?(db, opts) }
+
+      query, relations = root_query
+      # A Union reads its Vocabulary upstreams with one combined query, so the
+      # root query has no CTE for them. Their own CTEs join the WITH list.
+      # The second root_query evaluates the tree again. Reusing the first
+      # evaluation would give the root CTEs other names (each evaluation
+      # takes new numbers from the scope), and the SQL would change.
+      unless relations.all?
+        extra = @operators.zip(relations).filter_map { |op, name| op.evaluate(db) unless name }
+        query, relations = root_query(extra)
+        return unless relations.all?
+      end
+
+      query.from(relations.each_with_index.map { |name, i| count_query(name, i) }
+                          .reduce { |all, one| all.union(one, all: true, from_self: false) })
+    end
+
+    # The root query, and the name of the CTE in its WITH list that holds the
+    # rows of each operator (nil when it has none). An operator's CTE that
+    # Scope#merge_identical_ctes removed is read through the CTE that
+    # replaced it.
+    def root_query(extra = [])
+      query = scope.with_ctes(root, db, extra: extra)
+      with_names = Array(query.opts[:with]).map { |w| w[:name] }
+      merged = query.opts.fetch(Scope::MERGED_CTES, {})
+      relations = @operators.map do |op|
+        scope.cte_names(op).reverse_each.map { |name| merged.fetch(name, name) }.find { |name| with_names.include?(name) }
+      end
+      [ query, relations ]
+    end
+
+    # Every operator that #annotate visits: the statement's tree, and the
+    # labeled operators that Recall operators stand in for.
+    def operators
+      found = []
+      visit = lambda do |op|
+        next if op.nil? || found.any? { |known| known.equal?(op) }
+
+        found << op
+        op.all_upstreams.each(&visit)
+      end
+      visit.call(root)
+      scope.known_operators.each_value(&visit)
+      found
+    end
+
+    def count_query(name, index)
+      db[name].select_group(:criterion_domain)
+              .select_append(Sequel.as(index, OPERATOR_COLUMN))
+              .select_append { Sequel.function(:count, 1).as(:rows) }
+              .select_append { count(:person_id).distinct.as(:n) }
+    end
+  end
+end

data/lib/conceptql/operators/co_reported.rb

@@ -17,7 +17,10 @@ module ConceptQL
         vocabularies.length > 1
       end
 
-      desc "Passes along all events that were co-reported in the same record in the source data."
+      desc "Passes along the events of its upstream streams that were reported in the same source record " \
+           "(the same context). A record is kept only if every upstream stream " \
+           "has at least one event in it. If any one upstream has no event in a record, the events of all " \
+           "streams in that record are dropped, without a warning."
 
       allows_many_upstreams
       category "Combine Streams"
@@ -27,14 +30,6 @@ module ConceptQL
       require_column :context_id
 
       def query(db)
-        if gdm?
-          gdm(db)
-        else
-          events_with_common_visits(db).inject { |q, query| q.union(query, all: true) }
-        end
-      end
-
-      def gdm(db)
         contexteds = upstreams.map do |stream|
           [ stream.cte_name(:contextified), contextify(db, stream) ]
         end.to_h
@@ -84,28 +79,6 @@ module ConceptQL
         # .select_append(Sequel[:c][:context_id].as(:context_id))
         # .from_self
       end
-
-      private
-
-      def visit_occurrence_ids_in_common(db)
-        @visit_occurrence_ids_in_common ||= upstream_queries(db).map do |q|
-          q.select(:visit_occurrence_id)
-        end.inject do |q, query|
-          q.from_self(alias: :tab1)
-           .join(query.as(:tab2), visit_occurrence_id: :visit_occurrence_id)
-           .select(Sequel[:tab1][:visit_occurrence_id].as(:visit_occurrence_id))
-        end
-      end
-
-      def events_with_common_visits(db)
-        upstream_queries(db).map { |q| q.where(visit_occurrence_id: visit_occurrence_ids_in_common(db)).from_self }
-      end
-
-      def upstream_queries(db)
-        @upstream_queries ||= upstreams.map do |expression|
-          expression.evaluate(db).select(*query_cols).from_self
-        end
-      end
     end
   end
 end

data/lib/conceptql/operators/count.rb

@@ -8,6 +8,10 @@ module ConceptQL
       register __FILE__
 
       desc "Counts the number of records that exactly match across all columns."
+      # Every stream is a set, so count returns 1 for nearly every row
+      # (seed conceptql-e8n, ruled 2026-10-30).
+      deprecated "count will become an error in a later release. To count the events of each person, " \
+                 'use ["sum", ["numeric", 1, <stream>], {"group_by_person": true}].'
       allows_one_upstream
       validate_one_upstream
       validate_no_arguments

data/lib/conceptql/operators/death.rb

@@ -9,7 +9,7 @@ module ConceptQL
 
       register __FILE__
 
-      desc "Selects all death records"
+      desc "Selects all death records. With an upstream stream, the operator selects the death record of each person in that stream."
       domains :death
       allows_one_upstream
 

data/lib/conceptql/operators/dedup.rb

@@ -7,7 +7,7 @@ module ConceptQL
     class Dedup < PassThru
       register __FILE__
 
-      desc "Counts the number of records that exactly match across all columns."
+      desc "Passes along each distinct record once. Records that have the same value in every column become one record."
       allows_one_upstream
       validate_one_upstream
 

data/lib/conceptql/operators/episode.rb

@@ -52,10 +52,13 @@ module ConceptQL
         # Get allowed episode gap
         gap_of = get_episode_gap
 
+        # episode_end_date is the latest end so far. The lag gives it for the
+        # records before this one, so a record starts a new episode when it
+        # starts more than gap_of days after the end of the records before it.
         gap_check = date_adjust_add(db,
                                     Sequel.function(:lag, :episode_end_date).over(partition: partition_vars, order: %i[start_date end_date]), gap_of, "days")
 
-        cond_gap_lt_start = (gap_check < Sequel[:episode_end_date]) | (nil)
+        cond_gap_lt_start = (gap_check < Sequel[:start_date]) | (nil)
 
         dt2 = dt1.from_self.select_append(
           gap_of.as(:gap_of),
@@ -98,7 +101,12 @@ module ConceptQL
           Sequel.function(:max, :episode_end_date).as(:end_date)
         )
 
-        # Episodes streams return null for criterion_id,table, and domain which messes up uuid generation so we had to add constants to these values
+        # An episode has no criterion_id, criterion_table or criterion_domain
+        # of its own, so these constants name it in every output row. They
+        # were added because a NULL part made the uuid NULL. The uuid now
+        # gives a NULL part as an empty string (bead conceptql-18n), but the
+        # constants stay, because they are part of each episode row: the
+        # expected results in test/results/gdm/episode/ show them.
         episode_summary = episode_summary.from_self.select_append(
           Sequel[0].cast_numeric.as(:criterion_id),
           Sequel.cast_string("episode").as(:criterion_table),

data/lib/conceptql/operators/ethnicity.rb

@@ -15,10 +15,6 @@ module ConceptQL
       validate_no_upstreams
       validate_at_least_one_argument
 
-      def query_cols
-        table_columns(table)
-      end
-
       def table
         source_table
       end

data/lib/conceptql/operators/except.rb

@@ -10,6 +10,11 @@ module ConceptQL
       desc "Removes all incoming records that appear in the right hand set of records."
       default_query_columns
 
+      # An `as` option would cast both streams before the comparison. Except
+      # has no such option, and a statement that sets one fails validation
+      # instead of running as a plain except (bead conceptql-dlq).
+      unsupported_options :as
+
       def query(db)
         cols_to_compare = %i[
           criterion_id

data/lib/conceptql/operators/filter.rb

@@ -7,9 +7,14 @@ module ConceptQL
     class Filter < BinaryOperatorOperator
       register __FILE__
 
-      desc "Keeps left hand records that share the same person, criterion_id, and criterion_domain with a right hand side record."
+      desc "Keeps left hand records that share the same person, criterion_id, criterion_table, and criterion_domain with a right hand side record."
       default_query_columns
 
+      # An `as` option would cast both streams before the comparison. Filter
+      # has no such option, and a statement that sets one fails validation
+      # instead of running as a plain filter (bead conceptql-8ac).
+      unsupported_options :as
+
       def query(db)
         rhs = right.evaluate(db)
         rhs = rhs.from_self.select_group(*columns)
@@ -26,8 +31,13 @@ module ConceptQL
         columns.map { |c| [ Sequel[:l][c], Sequel[:r][c] ] }
       end
 
+      # criterion_id is an id within its criterion_table, so two records from
+      # different tables can share one. Without criterion_table, a condition
+      # from observations matched a hospitalization from collections (both
+      # have the condition_occurrence domain) whenever their ids were equal
+      # (bead conceptql-fun).
       def determine_columns
-        columns = %w[person_id criterion_id criterion_domain]
+        columns = %w[person_id criterion_id criterion_table criterion_domain]
         columns += %w[start_date end_date] unless options[:ignore_dates]
         columns.map(&:to_sym)
       end