conceptql 0.5.0 → 0.6.0

data/lib/conceptql/operators/trim_date.rb

@@ -14,6 +14,18 @@ module ConceptQL
     #
     # If the RHS record's start_date is later than the LHS end_date, the LHS
     # record is passed thru unaffected.
+    #
+    # With join_columns (only t_shank gives it), "per person" means per
+    # person and join_columns values, with or without compare_all.
+    #
+    # With compare_all (only t_shank gives it), each LHS record is compared
+    # with every RHS record of the person and of the join_columns, not only
+    # with the person's first (trim_date_end) or last (trim_date_start) RHS
+    # record. Each LHS record comes out once, trimmed by the nearest RHS
+    # record that passes: the earliest start_date on or after its start_date
+    # (trim_date_end), or the latest end_date on or before its end_date
+    # (trim_date_start). An LHS record is discarded only when it has RHS
+    # records and none of them passes.
     class TrimDate < TemporalOperator
       desc <<~EOF
         Trims the end_date of the left hand records (LHR) by the earliest
@@ -38,6 +50,8 @@ module ConceptQL
                .left_join(Sequel.as(right_stream_query(db), :r), join_columns.inject(&:&))
                .where(where_criteria)
 
+        ds = nearest_trim_date(ds) if compare_all?
+
         ds = dm.selectify(ds, qualifier: :l, replace: replacement_columns)
 
         ds = apply_selectors(ds)
@@ -51,16 +65,48 @@ module ConceptQL
         options[:compare_all]
       end
 
+      # With compare_all, each left row joins every right row of the person
+      # (and of the join columns), and the where clause keeps the right rows
+      # that pass. Group those back to one row for each left row, and keep
+      # the nearest passing trim date (trim_function). A left row without a
+      # right row has a NULL trim date, so it passes through unchanged (bead
+      # conceptql-x6m, seed conceptql-ysx). Every stream is a set, so left
+      # rows that are equal in every column come out once (seed conceptql-0eb).
+      def nearest_trim_date(ds)
+        ds.select_group(*dynamic_columns.map { |column| Sequel[:l][column] })
+          .select_append(Sequel.function(trim_function, trim_bound).as(:trim_bound))
+          .from_self(alias: :l)
+      end
+
+      # The trim date of the joined right row, or the nearest one of the left
+      # row when compare_all grouped the right rows.
+      def trim_date_column
+        compare_all? ? Sequel[:l][:trim_bound] : trim_bound
+      end
+
       def right_stream_query(db)
-        rh = if compare_all?
-               super
-        else
-               occ = to_op([ :occurrence, occurrence_number, right ])
-               occ.evaluate(db)
-        end
+        rh = compare_all? ? super : nearest_right_row(db)
         columnizer.apply(rh)
       end
 
+      # Without compare_all, the first (trim_date_end) or last
+      # (trim_date_start) right row of each person and join_columns values,
+      # the grouping compare_all uses. The rows are ordered as the occurrence
+      # operator orders them: by start_date, then criterion_id. The occurrence
+      # operator groups by person only, so a left row whose join values
+      # differed from those of the person's first right row passed through
+      # untrimmed (bead conceptql-18n).
+      def nearest_right_row(db)
+        direction = occurrence_number.negative? ? :desc : :asc
+        right.evaluate(db)
+             .select_append(Sequel[:ROW_NUMBER].function.over(
+               partition: join_columns_option.uniq,
+               order: %i[start_date criterion_id].map { |column| Sequel.send(direction, column) }
+             ).as(:rn))
+             .from_self
+             .where(rn: occurrence_number.abs)
+      end
+
       def columnizer
         columnizer = Columnizer.new
         columnizer.add_columns(:person_id, trim_date, *join_columns_option)

data/lib/conceptql/operators/trim_date_end.rb

@@ -32,7 +32,17 @@ module ConceptQL
       end
 
       def replacement_columns
-        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, within_start, l_end_date)) }
+        { end_date: Sequel.function(:least, l_end_date, Sequel.function(:coalesce, trim_date_column, l_end_date)) }
+      end
+
+      # The date that trims a left row, from one right row.
+      def trim_bound
+        within_start
+      end
+
+      # With compare_all, the nearest trim date of the right rows that pass.
+      def trim_function
+        :min
       end
 
       def occurrence_number

data/lib/conceptql/operators/trim_date_start.rb

@@ -32,7 +32,17 @@ module ConceptQL
       end
 
       def replacement_columns
-        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, within_end, l_start_date)) }
+        { start_date: Sequel.function(:greatest, l_start_date, Sequel.function(:coalesce, trim_date_column, l_start_date)) }
+      end
+
+      # The date that trims a left row, from one right row.
+      def trim_bound
+        within_end
+      end
+
+      # With compare_all, the nearest trim date of the right rows that pass.
+      def trim_function
+        :max
       end
 
       def occurrence_number

data/lib/conceptql/operators/union.rb

@@ -70,11 +70,55 @@ module ConceptQL
 
         queries += Combiner.new(combinables).queries(db)
 
+        all = disjoint_upstreams?
         queries.map! { |ds| ds.from_self.select(*query_cols).from_self }
         queries.inject do |q, query|
-          q.union(query, all: true)
+          q.union(query, all: all)
         end
       end
+
+      # The (criterion_domain, vocabulary condition) pairs of every row this
+      # union can emit, or nil when an upstream does not fix them at build
+      # time. See #disjoint_upstreams?.
+      def branches
+        upstream_branches = upstreams.map { |upstream| branches_of(upstream) }
+        upstream_branches.flatten(1) if upstream_branches.all?
+      end
+
+      private
+
+      # Every stream is a set (seed conceptql-0eb), so union removes the rows
+      # that arrive from more than one upstream: UNION, not UNION ALL. The
+      # duplicate removal is expensive on large data (conceptql-5pz), so union
+      # skips it when no row can arrive from two of its queries. It proves
+      # that only for vocabulary operators, cpt_or_hcpcs-style operators and
+      # unions of them, because each of their rows carries the operator's
+      # domain as criterion_domain and comes from a row of its vocabulary.
+      # Two rows with different domains differ, and so do two rows of one
+      # domain (one table) from different vocabularies of one column. Any
+      # other upstream can overlap anything, so union removes duplicates.
+      def disjoint_upstreams?
+        combinables, individuals = upstreams.partition { |upstream| upstream.is_a?(Vocabulary) }
+        queries = individuals.map { |upstream| branches_of(upstream) }
+        queries += combinables.group_by(&:domain).values.map { |combos| combos.flat_map { |c| branches_of(c) } }
+        return false unless queries.all?
+
+        queries.combination(2).all? do |one, other|
+          one.product(other).all? { |a, b| disjoint_branches?(a, b) }
+        end
+      end
+
+      def branches_of(upstream)
+        case upstream
+        when Vocabulary then [ [ upstream.domain, upstream.vocabulary_condition ] ]
+        when MultipleVocabularies then upstream.vocab_ops.flat_map { |vo| branches_of(vo) }
+        when Union then upstream.branches
+        end
+      end
+
+      def disjoint_branches?((domain, condition), (other_domain, other_condition))
+        domain != other_domain || (condition.keys == other_condition.keys && condition != other_condition)
+      end
     end
   end
 end

data/lib/conceptql/operators/vocabulary.rb

@@ -38,7 +38,7 @@ module ConceptQL
         args -= bad_arguments
 
         missing_args = args - dm.known_codes(db, vocabulary_id, args)
-        add_warning("unknown code(s)", *missing_args) unless missing_args.empty?
+        add_warning(UNKNOWN_CODE, *missing_args) unless missing_args.empty?
 
         warn_about_reused_codes(db, args - missing_args)
       end
@@ -137,13 +137,17 @@ module ConceptQL
         unless defined?(@code_regexp)
           @code_regexp = nil
 
-          if (reg_str = vocab_entry.format_regexp)
-            @code_regexp = Regexp.union(Regexp.new(reg_str, Regexp::IGNORECASE), /\A\*\Z/)
+          if (format = Vocabularies::Formats.regexp(vocab_entry.omopv5_id))
+            @code_regexp = Regexp.union(format, /\A\*\Z/)
           end
         end
         @code_regexp
       end
 
+      def bad_format_message
+        vocab_entry.omopv5_id == "NDC" ? NDC_BAD_FORMAT : BAD_FORMAT
+      end
+
       def vocabulary_id
         @vocabulary_id ||= translated_vocabulary_id
       end

data/lib/conceptql/paths.rb

@@ -22,6 +22,16 @@ module ConceptQL
       ConceptQL.config_dir / "vocabularies.csv"
     end
 
+    # A pinned copy of vocabulary_formats.tsv (ConceptQL::Vocabularies::Formats)
+    def vocabulary_formats_file_path
+      ConceptQL.config_dir / "vocabulary_formats.tsv"
+    end
+
+    # The vocabulary_formats commit that the copy came from
+    def vocabulary_formats_sha_file_path
+      ConceptQL.config_dir / "vocabulary_formats.sha"
+    end
+
     def multiple_vocabularies_file_path
       ConceptQL.config_dir / "multiple_vocabularies.csv"
     end

data/lib/conceptql/query.rb

@@ -4,6 +4,7 @@ require "json"
 require "forwardable"
 require_relative "scope"
 require_relative "nodifier"
+require_relative "one_query_counts"
 require "sequel/sqlformat"
 
 module ConceptQL
@@ -20,29 +21,39 @@ module ConceptQL
         [ super, "OG ERROR:", og_message ].join("\n")
       end
     end
+  end
 
-    # Raised by #code_list when the statement has validation errors: the
-    # operator tree of an invalid statement can have missing operands, and a
-    # partial code list would look complete. #errors is the errors hash of
-    # #scope_annotate (operator name => list of [message, *details]).
-    class InvalidStatementError < QueryError
-      attr_reader :statement, :errors
-
-      def initialize(statement, errors)
-        @statement = statement
-        @errors = errors
-        super([ "Invalid ConceptQL statement, so it has no code list. Validation errors:",
-               *errors.flat_map { |op, op_errors| op_errors.map { |e| "  #{op}: #{describe(e)}" } },
-               "Statement: #{statement.inspect}" ].join("\n"))
-      end
+  # Raised when a Query builds the SQL of a statement (#query, #sql,
+  # #sql_statements, and #all, #count, #execute, #order and #analyze through
+  # #query) or its code list (#code_list), and the statement has validation
+  # errors. An invalid statement can crash while it builds, or build SQL that
+  # silently returns fewer rows. Warnings do not raise.
+  #
+  # #errors is the errors hash that #scope_annotate reports (operator name =>
+  # list of [message, *details]), so it holds the errors that annotate shows.
+  # #statement is the statement that the Query holds.
+  class InvalidStatementError < Query::QueryError
+    attr_reader :statement, :errors
+
+    def initialize(statement, errors)
+      @statement = statement
+      @errors = errors
+      super([ "Invalid ConceptQL statement. Validation errors:",
+             *errors.flat_map { |op, op_errors| op_errors.map { |e| "  #{op}: #{describe(e)}" } },
+             "Statement: #{statement.inspect}" ].join("\n"))
+    end
 
-      private
+    private
 
-      def describe(error)
-        message, *details = error
-        details.empty? ? message.to_s : "#{message} (#{details.flatten.map(&:to_s).join(', ')})"
-      end
+    def describe(error)
+      message, *details = error
+      details.empty? ? message.to_s : "#{message} (#{details.flatten.map(&:to_s).join(', ')})"
     end
+  end
+
+  class Query
+    # The name of InvalidStatementError in 0.4.0 to 0.5.0.
+    InvalidStatementError = ConceptQL::InvalidStatementError
 
     extend Forwardable
     def_delegators :query, :all, :count, :execute, :order
@@ -66,7 +77,9 @@ module ConceptQL
       query(explain: true, analyze: true).analyze
     end
 
+    # Raises InvalidStatementError if the statement has validation errors.
     def query(opts = {})
+      validate!
       (@queries ||= {})[opts] ||= nodifier.scope.with_ctes(operator, db, opts)
     end
 
@@ -112,11 +125,26 @@ module ConceptQL
         end
       end
       Hash[stmts]
+    rescue InvalidStatementError
+      raise
     rescue StandardError
       raise QueryError, "Failed to generate SQL for #{statement.inspect}"
     end
 
+    # With counts on, DuckDB and Postgres count every operator with one SQL
+    # statement (OneQueryCounts). Operators memoize #annotate, so a later call
+    # with other opts returns the first result.
+    #
+    # The errors of the window table's own statement are reported under
+    # "window_table" (see #window_table_errors). Then nothing is counted,
+    # because every count joins the window table, and building its SQL raises.
     def annotate(opts = {})
+      window_errors = window_table_errors
+      unless window_errors.empty?
+        nodifier.scope.add_errors("window_table", window_errors)
+        opts = opts.merge(skip_counts: true)
+      end
+      nodifier.scope.one_query_counts = OneQueryCounts.new(operator, db, opts)
       operator.annotate(db, opts)
     end
 
@@ -164,12 +192,11 @@ module ConceptQL
       operator.rdbms
     end
 
-    # Raises InvalidStatementError if the statement has validation errors.
-    # Warnings do not raise.
+    # Raises InvalidStatementError if the statement has validation errors:
+    # the operator tree of an invalid statement can have missing operands,
+    # and a partial code list would look complete. Warnings do not raise.
     def code_list(_ignored_db = nil)
-      errors = validation_errors
-      raise InvalidStatementError.new(statement, errors) unless errors.empty?
-
+      validate!
       operator.code_list(db).uniq
     end
 
@@ -189,6 +216,33 @@ module ConceptQL
 
     attr_reader :cdb, :nodifier
 
+    # Raises InvalidStatementError if #validation_errors is not empty. The
+    # statement and the opts of a Query do not change, so it checks once.
+    # Warnings do not raise. cause: nil keeps an error that a caller is
+    # handling out of the message (QueryError#message prints the cause).
+    def validate!
+      return if @valid
+
+      errors = validation_errors
+      raise InvalidStatementError.new(statement, errors), cause: nil unless errors.empty?
+
+      @valid = true
+    end
+
+    # The validation errors of the window table's statement, when the window
+    # table is a ConceptQL statement (an Array, as Window::Table reads it), as
+    # one list of [message, *details]. Each message starts with the name of
+    # the operator that reported it. A table name has no errors.
+    def window_table_errors
+      scope_opts = nodifier.scope.opts
+      window_table = scope_opts.merge(scope_opts[:window_opts] || {})[:window_table]
+      return [] unless window_table.is_a?(Array)
+
+      cdb.query(window_table).validation_errors.flat_map do |op, op_errors|
+        op_errors.map { |message, *details| [ "#{op}: #{message}", *details ] }
+      end
+    end
+
     def extract_statement(stmt, opts)
       if !stmt.is_a?(Array)
         raise "Improper ConceptQL statement: Expected an Array, got a #{stmt.class}"

data/lib/conceptql/query_modifiers/gdm/provider_query_modifier.rb

@@ -29,20 +29,20 @@ module ConceptQL
                    Sequel[:p][:specialty_concept_id].as(:practitioner_specialty_concept_id)
                  )
           else
+            # The data model names the columns of the source table. ds.columns
+            # asked the database for them while the SQL was built (bead
+            # conceptql-6k8).
+            cols = dm.table_cols(source_table)
             ds = query.select_all
-            ds = if ds.columns.include?(:provider_id)
+            ds = if cols.include?(:provider_id)
                    ds
-            elsif dm.table_cols(source_table).include?(:practitioner_id)
+            elsif cols.include?(:practitioner_id)
                    ds.select_append(Sequel[:practitioner_id].as(:provider_id))
             else
                    ds.select_append(Sequel.cast(nil, :Bigint).as(:provider_id))
             end
             %i[role_type_concept_id context_specialty_concept_id practitioner_specialty_concept_id].each do |col|
-              ds = if ds.columns.include?(col)
-                     ds
-              else
-                     ds.select_append(Sequel.cast(nil, :Bigint).as(col))
-              end
+              ds = ds.select_append(Sequel.cast(nil, :Bigint).as(col)) unless cols.include?(col)
             end
             ds
           end.from_self

data/lib/conceptql/rdbms/generic.rb

@@ -53,9 +53,13 @@ module ConceptQL
           .inject(:+)
       end
 
+      # A NULL part is an empty string. The || operator gives NULL when one
+      # part is NULL, and a NULL uuid matches nothing (bead conceptql-18n).
+      # COALESCE keeps the place of each part, so "1//t/2008-01-01/" cannot
+      # be read as other parts. Postgres, DuckDB and Spark all support it.
       def uuid_items
         %w[person_id criterion_id criterion_table start_date end_date].map do |column|
-          Sequel.cast_string(column.to_sym)
+          Sequel.function(:coalesce, Sequel.cast_string(column.to_sym), "")
         end
       end
 
@@ -79,6 +83,22 @@ module ConceptQL
         false
       end
 
+      # True if an IN subquery may sit in a SELECT-list expression, such as
+      # the CASE flags of One In Two Out. Scope#with_ctes reads a long IN list
+      # from its list CTE there only when this is true.
+      def supports_in_subquery_in_select?
+        true
+      end
+
+      # True if a NOT IN subquery may sit in a nested condition, such as the
+      # "IS NULL OR NOT IN" of the gender operator. Scope#with_ctes reads a
+      # long NOT IN list from its list CTE only when this is true. It does not
+      # check whether a NOT IN condition is nested, so false keeps every long
+      # NOT IN list a NOT IN list.
+      def supports_nested_not_in_subquery?
+        true
+      end
+
       def create_options(_scope, _ds)
         opts = {}
         opts[:analyze] = opts[:explain] = explain_temp_tables?

data/lib/conceptql/rdbms/spark.rb

@@ -13,6 +13,18 @@ module ConceptQL
       def null_safe_equal(lhs, rhs)
         Sequel.lit("(? <=> ?)", lhs, rhs)
       end
+
+      # Spark accepts an IN subquery only in a filter or a join: "IN/EXISTS
+      # predicate sub-queries can only be used in Filter/Join" (Spark 3.3).
+      def supports_in_subquery_in_select?
+        false
+      end
+
+      # Spark rejects a NOT IN subquery in a nested condition: "Null-aware
+      # predicate sub-queries cannot be used in nested conditions" (Spark 3.3).
+      def supports_nested_not_in_subquery?
+        false
+      end
     end
   end
 end