conceptql 0.4.1 → 0.6.0

data/lib/conceptql/operators/read.rb

@@ -18,25 +18,15 @@ module ConceptQL
       conceptql_spec_id "vocabulary"
 
       def query(db)
-        gdm? ? gdm(db) : omopv4(db)
-      end
-
-      def gdm(db)
         ops(db).first.evaluate(db)
       end
 
-      def omopv4(db)
-        streams = ops(db).map { |op| op.evaluate(db) }
-
-        streams.inject { |q, query| q.union(query, all: true) }.from_self
-      end
-
       def domains(db)
         ops.first.domains(db)
       end
 
       def query_cols
-        gdm? ? ops.first.query_cols : ops.map(&:query_cols).flatten.uniq
+        ops.first.query_cols
       end
 
       def required_columns
@@ -53,40 +43,11 @@ module ConceptQL
         ops.flat_map { |op| op.describe_codes(db, codes) }
       end
 
-      def klasses
-        @klasses ||= {
-          condition_occurrence: ReadCondition,
-          procedure_occurrence: ReadProcedure,
-          drug_exposure: ReadDrug,
-          observation: ReadObservation
-        }
-      end
-
       class ReadBase < ConceptQL::Operators::Vocabulary
         preferred_name "READ"
         argument :read_codes, type: :codelist, vocab: "Read"
       end
 
-      class ReadOmopBase < ReadBase
-        include ConceptQL::Vocabularies::Behaviors::Omopish
-
-        def vocabulary_id
-          17
-        end
-
-        def source_table
-          table
-        end
-
-        def domain
-          table
-        end
-
-        def domains(_db)
-          Array(domain)
-        end
-      end
-
       class ReadGDM < ReadBase
         include ConceptQL::Vocabularies::Behaviors::Gdmish
         include ConceptQL::Behaviors::Labish
@@ -103,66 +64,6 @@ module ConceptQL
           [ :condition_occurrence ]
         end
       end
-
-      class ReadCondition < ReadOmopBase
-        def table
-          :procedure_occurrence
-        end
-
-        def source_column
-          :procedure_source_value
-        end
-
-        def concept_column
-          :procedure_concept_id
-        end
-      end
-
-      class ReadProcedure < ReadOmopBase
-        def table
-          :procedure_occurrence
-        end
-
-        def source_column
-          :procedure_source_value
-        end
-
-        def concept_column
-          :procedure_concept_id
-        end
-      end
-
-      class ReadObservation < ReadOmopBase
-        include ConceptQL::Behaviors::Labish
-
-        def table
-          :observation
-        end
-
-        def source_column
-          :observation_source_value
-        end
-
-        def concept_column
-          :observation_concept_id
-        end
-      end
-
-      class ReadDrug < ReadOmopBase
-        include ConceptQL::Behaviors::Drugish
-
-        def table
-          :drug_exposure
-        end
-
-        def source_column
-          :drug_source_value
-        end
-
-        def concept_column
-          :drug_concept_id
-        end
-      end
     end
   end
 end

data/lib/conceptql/operators/recall.rb

@@ -14,7 +14,7 @@ module ConceptQL
     class Recall < Operator
       register __FILE__
 
-      desc "Recalls a set of records from a labeled operator in the statement"
+      desc "Passes along the records of the operator in the statement that has the given label."
 
       argument :name, type: :string
       category "Get Related Data"

data/lib/conceptql/operators/snf.rb

@@ -5,14 +5,13 @@ require_relative "operator"
 module ConceptQL
   module Operators
     class Snf < Operator
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
       include ConceptQL::Behaviors::Utilizable
 
-      desc "Selects admission records of type 'SNF'"
+      desc "Selects skilled nursing facility (SNF) admission records."
 
       def collection_type
-        "SNF"
+        "snf"
       end
     end
   end

data/lib/conceptql/operators/sum.rb

@@ -8,7 +8,12 @@ module ConceptQL
       register __FILE__
 
       def events_per_patient
-        :multiple
+        group_by_person? ? :single : :multiple
+      end
+
+      # A person result per person, as the person cast gives (seed conceptql-6c3)
+      def domains(db)
+        group_by_person? ? [ :person ] : super
       end
 
       def multiple_vocabularies
@@ -17,16 +22,23 @@ module ConceptQL
         vocabularies.length > 1
       end
 
-      desc <<~EOF
-        Sums value_as_number across all records that match on all but start_date, end_date.
-        For start_date and end_date the min and max of each respectively is returned.'
-      EOF
+      desc "Groups the records that have the same values in all columns except criterion_id, start_date, " \
+           "end_date and value_as_number. For each group, the operator passes along one record " \
+           "with the sum of value_as_number, the earliest start_date, the most recent end_date " \
+           "and a criterion_id of 0. With group_by_person, the operator groups all of the records " \
+           "of each person and passes along one person record for each person."
+      option :group_by_person, type: :boolean,
+                               instructions: "Choose whether to sum all of the records of each person into one person record, for example to count the events of each person"
       default_query_columns
       validate_at_least_one_upstream
       validate_no_arguments
+      # The editor's checkbox sends true or false, and an unset field nil
+      validate_option ->(value) { [ true, false, nil ].include?(value) }, :group_by_person
       require_column :value_as_number
 
       def query(db)
+        return per_person_query(db) if group_by_person?
+
         db.from(unioned(db))
           .select_group(*(dynamic_columns - %i[start_date end_date criterion_id value_as_number]))
           .select_append(Sequel.lit("?", 0).as(:criterion_id))
@@ -36,6 +48,35 @@ module ConceptQL
           .from_self
       end
 
+      # One row for each person (and window): the earliest start_date, the
+      # latest end_date and the sum of value_as_number. criterion_table,
+      # criterion_domain and column_family are those of the person cast. No
+      # single record stands behind the row, so criterion_id is 0 and the
+      # other columns are NULL.
+      def per_person_query(db)
+        person = {
+          criterion_id: Sequel.lit("?", 0),
+          criterion_table: Sequel.cast_string(dm.table_by_domain(:person).to_s),
+          criterion_domain: Sequel.cast_string("person"),
+          column_family: Sequel.cast_string(ConceptQL::Scope::DEFAULT_COLUMN_FAMILY)
+        }
+        aggregated = %i[start_date end_date value_as_number]
+        nulls = dynamic_columns - matching_columns - person.keys - aggregated
+
+        db.from(unioned(db))
+          .select_group(*matching_columns)
+          .select_append(*person.map { |column, value| value.as(column) })
+          .select_append { min(start_date).as(:start_date) }
+          .select_append { max(end_date).as(:end_date) }
+          .select_append { sum(value_as_number).as(:value_as_number) }
+          .select_append(*nulls.map { |column| rdbms.process(column, nil) })
+          .from_self
+      end
+
+      def group_by_person?
+        options[:group_by_person]
+      end
+
       def unioned(db)
         upstreams.map { |c| c.evaluate(db).select(*query_cols) }.inject do |uni, q|
           uni.union(q.from_self)

data/lib/conceptql/operators/temporal_operator.rb

@@ -99,8 +99,12 @@ module ConceptQL
       def apply_selectors(ds, opts = {})
         return ds unless include_rhs_columns
 
-        ds = ds.select_remove(*include_rhs_columns).select_append(*rhs_columns(opts)) if include_rhs_columns
-        ds
+        # A dataset without a select list reads an upstream, so it has the
+        # scope's columns. Name them: select_remove would ask the database
+        # for them, and the CTE of a labeled upstream does not exist while
+        # the SQL is built (bead conceptql-6k8).
+        ds = ds.select(*dynamic_columns) if ds.opts[:select].blank?
+        ds.select_remove(*include_rhs_columns).select_append(*rhs_columns(opts))
       end
 
       def rhs_columns(opts)

data/lib/conceptql/operators/trim_date.rb

@@ -14,6 +14,18 @@ module ConceptQL
     #
     # If the RHS record's start_date is later than the LHS end_date, the LHS
     # record is passed thru unaffected.
+    #
+    # With join_columns (only t_shank gives it), "per person" means per
+    # person and join_columns values, with or without compare_all.
+    #
+    # With compare_all (only t_shank gives it), each LHS record is compared
+    # with every RHS record of the person and of the join_columns, not only
+    # with the person's first (trim_date_end) or last (trim_date_start) RHS
+    # record. Each LHS record comes out once, trimmed by the nearest RHS
+    # record that passes: the earliest start_date on or after its start_date
+    # (trim_date_end), or the latest end_date on or before its end_date
+    # (trim_date_start). An LHS record is discarded only when it has RHS
+    # records and none of them passes.
     class TrimDate < TemporalOperator
       desc <<~EOF
         Trims the end_date of the left hand records (LHR) by the earliest
@@ -38,6 +50,8 @@ module ConceptQL
                .left_join(Sequel.as(right_stream_query(db), :r), join_columns.inject(&:&))
                .where(where_criteria)
 
+        ds = nearest_trim_date(ds) if compare_all?
+
         ds = dm.selectify(ds, qualifier: :l, replace: replacement_columns)
 
         ds = apply_selectors(ds)
@@ -51,16 +65,48 @@ module ConceptQL
         options[:compare_all]
       end
 
+      # With compare_all, each left row joins every right row of the person
+      # (and of the join columns), and the where clause keeps the right rows
+      # that pass. Group those back to one row for each left row, and keep
+      # the nearest passing trim date (trim_function). A left row without a
+      # right row has a NULL trim date, so it passes through unchanged (bead
+      # conceptql-x6m, seed conceptql-ysx). Every stream is a set, so left
+      # rows that are equal in every column come out once (seed conceptql-0eb).
+      def nearest_trim_date(ds)
+        ds.select_group(*dynamic_columns.map { |column| Sequel[:l][column] })
+          .select_append(Sequel.function(trim_function, trim_bound).as(:trim_bound))
+          .from_self(alias: :l)
+      end
+
+      # The trim date of the joined right row, or the nearest one of the left
+      # row when compare_all grouped the right rows.
+      def trim_date_column
+        compare_all? ? Sequel[:l][:trim_bound] : trim_bound
+      end
+
       def right_stream_query(db)
-        rh = if compare_all?
-               super
-        else
-               occ = to_op([ :occurrence, occurrence_number, right ])
-               occ.evaluate(db)
-        end
+        rh = compare_all? ? super : nearest_right_row(db)
         columnizer.apply(rh)
       end
 
+      # Without compare_all, the first (trim_date_end) or last
+      # (trim_date_start) right row of each person and join_columns values,
+      # the grouping compare_all uses. The rows are ordered as the occurrence
+      # operator orders them: by start_date, then criterion_id. The occurrence
+      # operator groups by person only, so a left row whose join values
+      # differed from those of the person's first right row passed through
+      # untrimmed (bead conceptql-18n).
+      def nearest_right_row(db)
+        direction = occurrence_number.negative? ? :desc : :asc
+        right.evaluate(db)
+             .select_append(Sequel[:ROW_NUMBER].function.over(
+               partition: join_columns_option.uniq,
+               order: %i[start_date criterion_id].map { |column| Sequel.send(direction, column) }
+             ).as(:rn))
+             .from_self
+             .where(rn: occurrence_number.abs)
+      end
+
       def columnizer
         columnizer = Columnizer.new
         columnizer.add_columns(:person_id, trim_date, *join_columns_option)

data/lib/conceptql/operators/trim_date_end.rb

@@ -32,7 +32,17 @@ module ConceptQL
       end
 
       def replacement_columns
-        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, within_start, l_end_date)) }
+        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, trim_date_column, l_end_date)) }
+      end
+
+      # The date that trims a left row, from one right row.
+      def trim_bound
+        within_start
+      end
+
+      # With compare_all, the nearest trim date of the right rows that pass.
+      def trim_function
+        :min
       end
 
       def occurrence_number

data/lib/conceptql/operators/trim_date_start.rb

@@ -32,7 +32,17 @@ module ConceptQL
       end
 
       def replacement_columns
-        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, within_end, l_start_date)) }
+        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, trim_date_column, l_start_date)) }
+      end
+
+      # The date that trims a left row, from one right row.
+      def trim_bound
+        within_end
+      end
+
+      # With compare_all, the nearest trim date of the right rows that pass.
+      def trim_function
+        :max
       end
 
       def occurrence_number

data/lib/conceptql/operators/union.rb

@@ -70,11 +70,55 @@ module ConceptQL
 
         queries += Combiner.new(combinables).queries(db)
 
+        all = disjoint_upstreams?
         queries.map! { |ds| ds.from_self.select(*query_cols).from_self }
         queries.inject do |q, query|
-          q.union(query, all: true)
+          q.union(query, all: all)
         end
       end
+
+      # The (criterion_domain, vocabulary condition) pairs of every row this
+      # union can emit, or nil when an upstream does not fix them at build
+      # time. See #disjoint_upstreams?.
+      def branches
+        upstream_branches = upstreams.map { |upstream| branches_of(upstream) }
+        upstream_branches.flatten(1) if upstream_branches.all?
+      end
+
+      private
+
+      # Every stream is a set (seed conceptql-0eb), so union removes the rows
+      # that arrive from more than one upstream: UNION, not UNION ALL. The
+      # duplicate removal is expensive on large data (conceptql-5pz), so union
+      # skips it when no row can arrive from two of its queries. It proves
+      # that only for vocabulary operators, cpt_or_hcpcs-style operators and
+      # unions of them, because each of their rows carries the operator's
+      # domain as criterion_domain and comes from a row of its vocabulary.
+      # Two rows with different domains differ, and so do two rows of one
+      # domain (one table) from different vocabularies of one column. Any
+      # other upstream can overlap anything, so union removes duplicates.
+      def disjoint_upstreams?
+        combinables, individuals = upstreams.partition { |upstream| upstream.is_a?(Vocabulary) }
+        queries = individuals.map { |upstream| branches_of(upstream) }
+        queries += combinables.group_by(&:domain).values.map { |combos| combos.flat_map { |c| branches_of(c) } }
+        return false unless queries.all?
+
+        queries.combination(2).all? do |one, other|
+          one.product(other).all? { |a, b| disjoint_branches?(a, b) }
+        end
+      end
+
+      def branches_of(upstream)
+        case upstream
+        when Vocabulary then [ [ upstream.domain, upstream.vocabulary_condition ] ]
+        when MultipleVocabularies then upstream.vocab_ops.flat_map { |vo| branches_of(vo) }
+        when Union then upstream.branches
+        end
+      end
+
+      def disjoint_branches?((domain, condition), (other_domain, other_condition))
+        domain != other_domain || (condition.keys == other_condition.keys && condition != other_condition)
+      end
     end
   end
 end

data/lib/conceptql/operators/vocabulary.rb

@@ -38,10 +38,23 @@ module ConceptQL
         args -= bad_arguments
 
         missing_args = args - dm.known_codes(db, vocabulary_id, args)
+        add_warning(UNKNOWN_CODE, *missing_args) unless missing_args.empty?
 
-        return if missing_args.empty?
+        warn_about_reused_codes(db, args - missing_args)
+      end
 
-        add_warning("unknown code(s)", *missing_args)
+      # A code that the vocabulary reused for a new meaning selects records of
+      # both meanings: DRG 410 selects chemotherapy stays before 2007-10-01 and
+      # biliary tract procedures after it (conceptql-ttk). Each item names the
+      # code and all of its names, so the user sees the problem without a
+      # vocabulary lookup.
+      def warn_about_reused_codes(db, codes)
+        meanings = dm.code_meanings(db, vocabulary_id, codes)
+        reused = codes.filter_map do |code|
+          names = meanings[code.downcase]
+          "#{code}: #{names.join(' | ')}" if names
+        end
+        add_warning("code(s) with more than one meaning", *reused) unless reused.empty?
       end
 
       def describe_codes(db, codes)
@@ -124,13 +137,17 @@ module ConceptQL
         unless defined?(@code_regexp)
           @code_regexp = nil
 
-          if (reg_str = vocab_entry.format_regexp)
-            @code_regexp = Regexp.union(Regexp.new(reg_str, Regexp::IGNORECASE), /\A\*\Z/)
+          if (format = Vocabularies::Formats.regexp(vocab_entry.omopv5_id))
+            @code_regexp = Regexp.union(format, /\A\*\Z/)
           end
         end
         @code_regexp
       end
 
+      def bad_format_message
+        vocab_entry.omopv5_id == "NDC" ? NDC_BAD_FORMAT : BAD_FORMAT
+      end
+
       def vocabulary_id
         @vocabulary_id ||= translated_vocabulary_id
       end

data/lib/conceptql/paths.rb

@@ -22,6 +22,16 @@ module ConceptQL
       ConceptQL.config_dir / "vocabularies.csv"
     end
 
+    # A pinned copy of vocabulary_formats.tsv (ConceptQL::Vocabularies::Formats)
+    def vocabulary_formats_file_path
+      ConceptQL.config_dir / "vocabulary_formats.tsv"
+    end
+
+    # The vocabulary_formats commit that the copy came from
+    def vocabulary_formats_sha_file_path
+      ConceptQL.config_dir / "vocabulary_formats.sha"
+    end
+
     def multiple_vocabularies_file_path
       ConceptQL.config_dir / "multiple_vocabularies.csv"
     end
@@ -37,5 +47,11 @@ module ConceptQL
     def race_file
       ConceptQL.config_dir / "race.yml"
     end
+
+    # The ConceptQL specification, shipped in the gem for applications that
+    # serve it to readers (the Jigsaw diagram editor serves it over MCP).
+    def spec_path
+      root / "doc" / "spec.md"
+    end
   end
 end

data/lib/conceptql/query.rb

@@ -4,6 +4,7 @@ require "json"
 require "forwardable"
 require_relative "scope"
 require_relative "nodifier"
+require_relative "one_query_counts"
 require "sequel/sqlformat"
 
 module ConceptQL
@@ -20,29 +21,39 @@ module ConceptQL
         [ super, "OG ERROR:", og_message ].join("\n")
       end
     end
+  end
 
-    # Raised by #code_list when the statement has validation errors: the
-    # operator tree of an invalid statement can have missing operands, and a
-    # partial code list would look complete. #errors is the errors hash of
-    # #scope_annotate (operator name => list of [message, *details]).
-    class InvalidStatementError < QueryError
-      attr_reader :statement, :errors
-
-      def initialize(statement, errors)
-        @statement = statement
-        @errors = errors
-        super([ "Invalid ConceptQL statement, so it has no code list. Validation errors:",
-               *errors.flat_map { |op, op_errors| op_errors.map { |e| "  #{op}: #{describe(e)}" } },
-               "Statement: #{statement.inspect}" ].join("\n"))
-      end
+  # Raised when a Query builds the SQL of a statement (#query, #sql,
+  # #sql_statements, and #all, #count, #execute, #order and #analyze through
+  # #query) or its code list (#code_list), and the statement has validation
+  # errors. An invalid statement can crash while it builds, or build SQL that
+  # silently returns fewer rows. Warnings do not raise.
+  #
+  # #errors is the errors hash that #scope_annotate reports (operator name =>
+  # list of [message, *details]), so it holds the errors that annotate shows.
+  # #statement is the statement that the Query holds.
+  class InvalidStatementError < Query::QueryError
+    attr_reader :statement, :errors
+
+    def initialize(statement, errors)
+      @statement = statement
+      @errors = errors
+      super([ "Invalid ConceptQL statement. Validation errors:",
+             *errors.flat_map { |op, op_errors| op_errors.map { |e| "  #{op}: #{describe(e)}" } },
+             "Statement: #{statement.inspect}" ].join("\n"))
+    end
 
-      private
+    private
 
-      def describe(error)
-        message, *details = error
-        details.empty? ? message.to_s : "#{message} (#{details.flatten.map(&:to_s).join(', ')})"
-      end
+    def describe(error)
+      message, *details = error
+      details.empty? ? message.to_s : "#{message} (#{details.flatten.map(&:to_s).join(', ')})"
     end
+  end
+
+  class Query
+    # The name of InvalidStatementError in 0.4.0 to 0.5.0.
+    InvalidStatementError = ConceptQL::InvalidStatementError
 
     extend Forwardable
     def_delegators :query, :all, :count, :execute, :order
@@ -66,7 +77,9 @@ module ConceptQL
       query(explain: true, analyze: true).analyze
     end
 
+    # Raises InvalidStatementError if the statement has validation errors.
     def query(opts = {})
+      validate!
       (@queries ||= {})[opts] ||= nodifier.scope.with_ctes(operator, db, opts)
     end
 
@@ -112,11 +125,26 @@ module ConceptQL
         end
       end
       Hash[stmts]
+    rescue InvalidStatementError
+      raise
     rescue StandardError
       raise QueryError, "Failed to generate SQL for #{statement.inspect}"
     end
 
+    # With counts on, DuckDB and Postgres count every operator with one SQL
+    # statement (OneQueryCounts). Operators memoize #annotate, so a later call
+    # with other opts returns the first result.
+    #
+    # The errors of the window table's own statement are reported under
+    # "window_table" (see #window_table_errors). Then nothing is counted,
+    # because every count joins the window table, and building its SQL raises.
     def annotate(opts = {})
+      window_errors = window_table_errors
+      unless window_errors.empty?
+        nodifier.scope.add_errors("window_table", window_errors)
+        opts = opts.merge(skip_counts: true)
+      end
+      nodifier.scope.one_query_counts = OneQueryCounts.new(operator, db, opts)
       operator.annotate(db, opts)
     end
 
@@ -164,12 +192,11 @@ module ConceptQL
       operator.rdbms
     end
 
-    # Raises InvalidStatementError if the statement has validation errors.
-    # Warnings do not raise.
+    # Raises InvalidStatementError if the statement has validation errors:
+    # the operator tree of an invalid statement can have missing operands,
+    # and a partial code list would look complete. Warnings do not raise.
     def code_list(_ignored_db = nil)
-      errors = validation_errors
-      raise InvalidStatementError.new(statement, errors) unless errors.empty?
-
+      validate!
       operator.code_list(db).uniq
     end
 
@@ -189,6 +216,33 @@ module ConceptQL
 
     attr_reader :cdb, :nodifier
 
+    # Raises InvalidStatementError if #validation_errors is not empty. The
+    # statement and the opts of a Query do not change, so it checks once.
+    # Warnings do not raise. cause: nil keeps an error that a caller is
+    # handling out of the message (QueryError#message prints the cause).
+    def validate!
+      return if @valid
+
+      errors = validation_errors
+      raise InvalidStatementError.new(statement, errors), cause: nil unless errors.empty?
+
+      @valid = true
+    end
+
+    # The validation errors of the window table's statement, when the window
+    # table is a ConceptQL statement (an Array, as Window::Table reads it), as
+    # one list of [message, *details]. Each message starts with the name of
+    # the operator that reported it. A table name has no errors.
+    def window_table_errors
+      scope_opts = nodifier.scope.opts
+      window_table = scope_opts.merge(scope_opts[:window_opts] || {})[:window_table]
+      return [] unless window_table.is_a?(Array)
+
+      cdb.query(window_table).validation_errors.flat_map do |op, op_errors|
+        op_errors.map { |message, *details| [ "#{op}: #{message}", *details ] }
+      end
+    end
+
     def extract_statement(stmt, opts)
       if !stmt.is_a?(Array)
         raise "Improper ConceptQL statement: Expected an Array, got a #{stmt.class}"