blacklight 7.14.1 → 7.17.0

data/lib/blacklight/solr/response/facets.rb

@@ -29,6 +29,7 @@ module Blacklight::Solr::Response::Facets
   # represents a facet; which is a field and its values
   class FacetField
     attr_reader :name, :items
+    attr_accessor :missing
 
     def initialize name, items, options = {}
       @name = name
@@ -52,6 +53,14 @@ module Blacklight::Solr::Response::Facets
       @options[:prefix] || solr_default_prefix
     end
 
+    def type
+      @options[:type] || 'terms'
+    end
+
+    def data
+      @options[:data] || {}
+    end
+
     def index?
       sort == 'index'
     end
@@ -90,7 +99,7 @@ module Blacklight::Solr::Response::Facets
   # Get all the Solr facet data (fields, queries, pivots) as a hash keyed by
   # both the Solr field name and/or by the blacklight field name
   def aggregations
-    @aggregations ||= {}.merge(facet_field_aggregations).merge(facet_query_aggregations).merge(facet_pivot_aggregations)
+    @aggregations ||= {}.merge(facet_field_aggregations).merge(facet_query_aggregations).merge(facet_pivot_aggregations).merge(json_facet_aggregations)
   end
 
   def facet_counts
@@ -159,19 +168,24 @@ module Blacklight::Solr::Response::Facets
       items = values.map do |value, hits|
         i = FacetItem.new(value: value, hits: hits)
 
-        # solr facet.missing serialization
+        # legacy solr facet.missing serialization
         if value.nil?
           i.label = I18n.t(:"blacklight.search.fields.facet.missing.#{facet_field_name}", default: [:"blacklight.search.facets.missing"])
           i.fq = "-#{facet_field_name}:[* TO *]"
+          i.missing = true
         end
 
         i
       end
 
       options = facet_field_aggregation_options(facet_field_name)
-      hash[facet_field_name] = FacetField.new(facet_field_name,
-                                              items,
-                                              options)
+      facet_field = FacetField.new(facet_field_name, items, options)
+
+      if values[nil]
+        facet_field.missing = items.find(&:missing)
+      end
+
+      hash[facet_field_name] = facet_field
 
       # alias all the possible blacklight config names..
       blacklight_config.facet_fields.select { |_k, v| v.field == facet_field_name }.each_key do |key|
@@ -180,23 +194,6 @@ module Blacklight::Solr::Response::Facets
     end
   end
 
-  def facet_field_aggregation_options(facet_field_name)
-    options = {}
-    options[:sort] = (params[:"f.#{facet_field_name}.facet.sort"] || params[:'facet.sort'])
-    if params[:"f.#{facet_field_name}.facet.limit"] || params[:"facet.limit"]
-      options[:limit] = (params[:"f.#{facet_field_name}.facet.limit"] || params[:"facet.limit"]).to_i
-    end
-
-    if params[:"f.#{facet_field_name}.facet.offset"] || params[:'facet.offset']
-      options[:offset] = (params[:"f.#{facet_field_name}.facet.offset"] || params[:'facet.offset']).to_i
-    end
-
-    if params[:"f.#{facet_field_name}.facet.prefix"] || params[:'facet.prefix']
-      options[:prefix] = (params[:"f.#{facet_field_name}.facet.prefix"] || params[:'facet.prefix'])
-    end
-    options
-  end
-
   ##
   # Aggregate Solr's facet_query response into the virtual facet fields defined
   # in the blacklight configuration
@@ -211,12 +208,28 @@ module Blacklight::Solr::Response::Facets
         Blacklight::Solr::Response::Facets::FacetItem.new(value: key, hits: hits, label: facet_field.query[key][:label])
       end
 
+      items += facet_query_aggregations_from_json(facet_field)
+
       items = items.sort_by(&:hits).reverse if facet_field.sort && facet_field.sort.to_sym == :count
 
       hash[field_name] = Blacklight::Solr::Response::Facets::FacetField.new field_name, items
     end
   end
 
+  def facet_query_aggregations_from_json(facet_field)
+    return [] unless self['facets']
+
+    salient_facet_queries = facet_field.query.map { |_k, x| x[:fq] }
+
+    relevant_facet_data = self['facets'].select { |k, _v| salient_facet_queries.include?(k) }.reject { |_key, data| data['count'].zero? }
+
+    relevant_facet_data.map do |key, data|
+      salient_fields = facet_field.query.select { |_key, val| val[:fq] == key }
+      facet_key = ((salient_fields.keys if salient_fields.respond_to? :keys) || salient_fields.first).first
+      Blacklight::Solr::Response::Facets::FacetItem.new(value: facet_key, hits: data[:count], label: facet_field.query[facet_key][:label])
+    end
+  end
+
   ##
   # Convert Solr's facet_pivot response into
   # a hash of Blacklight::Solr::Response::Facet::FacetField objects
@@ -244,4 +257,45 @@ module Blacklight::Solr::Response::Facets
 
     Blacklight::Solr::Response::Facets::FacetItem.new(value: lst[:value], hits: lst[:count], field: lst[:field], items: items, fq: parent_fq)
   end
+
+  def construct_json_nested_facet_fields(bucket, parent_fq = {})
+    bucket.select { |_, nested| nested.is_a?(Hash) && nested.key?('buckets') }.map do |facet_field_name, nested|
+      nested['buckets'].map do |subbucket|
+        i = Blacklight::Solr::Response::Facets::FacetItem.new(field: facet_field_name, value: subbucket['val'], hits: subbucket['count'], fq: parent_fq, data: subbucket)
+
+        i.items = construct_json_nested_facet_fields(subbucket, parent_fq.merge(key => subbucket['val'])) if has_json_nested_facets?(subbucket)
+        i
+      end
+    end.flatten
+  end
+
+  def has_json_nested_facets?(bucket)
+    bucket.any? { |_, nested| nested.is_a?(Hash) && nested.key?('buckets') }
+  end
+
+  def json_facet_aggregations
+    return {} unless self['facets']
+
+    self['facets'].each_with_object({}) do |(facet_field_name, data), hash|
+      next if facet_field_name == 'count'
+
+      items = (data['buckets'] || []).map do |bucket|
+        i = Blacklight::Solr::Response::Facets::FacetItem.new(value: bucket['val'], hits: bucket['count'], data: bucket)
+
+        i.items = construct_json_nested_facet_fields(bucket, facet_field_name => bucket['val']) if has_json_nested_facets?(bucket)
+
+        i
+      end
+
+      options = facet_field_aggregation_options(facet_field_name).merge(data: data)
+      facet_field = FacetField.new(facet_field_name, items, options)
+
+      facet_field.missing = Blacklight::Solr::Response::Facets::FacetItem.new(
+        hits: data.dig('missing', 'count'),
+        data: data['missing']
+      ) if data['missing']
+
+      hash[facet_field_name] = facet_field
+    end
+  end
 end

data/lib/blacklight/solr/response/params.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+module Blacklight::Solr::Response::Params
+  # From https://solr.apache.org/guide/8_8/json-request-api.html#supported-properties-and-syntax
+  QUERY_PARAMETER_TO_JSON_PARAMETER_MAPPING = {
+    q: :query,
+    fq: :filter,
+    start: :offset,
+    rows: :limit,
+    fl: :fields,
+    sort: :sort
+  }.freeze
+
+  def params
+    header['params'] || request_params
+  end
+
+  def start
+    search_builder&.start || single_valued_param(:start).to_i
+  end
+
+  def rows
+    search_builder&.rows || single_valued_param(:rows).to_i
+  end
+
+  def sort
+    search_builder&.sort || single_valued_param(:sort)
+  end
+
+  def facet_field_aggregation_options(facet_field_name)
+    defaults = {
+      sort: single_valued_param(:'facet.sort'),
+      limit: single_valued_param(:"facet.limit")&.to_i || 100,
+      offset: single_valued_param(:"facet.offset")&.to_i || 0,
+      prefix: single_valued_param(:"facet.prefix")
+    }
+
+    json_facet = json_params.dig('facet', facet_field_name)&.slice(:limit, :offset, :prefix, :sort)&.symbolize_keys || {}
+
+    param_facet = {
+      sort: single_valued_param(:"f.#{facet_field_name}.facet.sort"),
+      limit: single_valued_param(:"f.#{facet_field_name}.facet.limit")&.to_i,
+      offset: single_valued_param(:"f.#{facet_field_name}.facet.offset")&.to_i,
+      prefix: single_valued_param(:"f.#{facet_field_name}.facet.prefix")
+    }.reject { |_k, v| v.nil? }
+
+    options = defaults.merge(json_facet).merge(param_facet)
+    options[:sort] ||= options[:limit].positive? ? 'count' : 'index'
+
+    options
+  end
+
+  private
+
+  def search_builder
+    request_params if request_params.is_a?(Blacklight::SearchBuilder)
+  end
+
+  # Extract JSON Request API parameters from the response header or the request itself
+  def json_params
+    encoded_json_params = header&.dig('params', 'json')
+
+    return request_params['json'] || {} if encoded_json_params.blank?
+
+    @json_params ||= JSON.parse(encoded_json_params).with_indifferent_access
+  end
+
+  # Handle merging solr parameters from the myriad of ways they may be expressed by applying the single-value
+  # precedence logic:
+  #
+  # From https://solr.apache.org/guide/8_8/json-request-api.html#json-parameter-merging :
+  # When multiple parameter values conflict with one another a single value is chosen based on the following precedence rules:
+  # - Traditional query parameters (q, rows, etc.) take first precedence and are used over any other specified values.
+  # - json-prefixed query parameters are considered next.
+  # - Values specified in the JSON request body have the lowest precedence and are only used if specified nowhere else.
+  #
+  # @param [String] key the solr parameter to use
+  def single_valued_param(key)
+    json_key = QUERY_PARAMETER_TO_JSON_PARAMETER_MAPPING[key]
+
+    params[key] ||
+      params["json.#{key}"] ||
+      json_params[json_key || key] ||
+      json_params.dig(:params, key) ||
+      json_params.dig(:params, "json.#{key}")
+  end
+
+  # Merge together multi-valued solr parameters from the myriad of ways they may be expressed.
+  # Unlike single-valued parameters, this merges all the values across the params.
+  #
+  # @param [String] key the solr parameter to use
+  def multivalued_param(key)
+    json_key = QUERY_PARAMETER_TO_JSON_PARAMETER_MAPPING[key]
+
+    [
+      params[key],
+      params["json.#{key}"],
+      json_params[json_key || key],
+      json_params.dig(:params, key),
+      json_params.dig(:params, "json.#{key}")
+    ].select(&:present?).inject([]) do |memo, arr|
+      memo.concat(Array.wrap(arr))
+    end
+  end
+end

data/lib/blacklight/solr/search_builder_behavior.rb

@@ -8,7 +8,8 @@ module Blacklight::Solr
         :default_solr_parameters, :add_query_to_solr, :add_facet_fq_to_solr,
         :add_facetting_to_solr, :add_solr_fields_to_query, :add_paging_to_solr,
         :add_sorting_to_solr, :add_group_config_to_solr,
-        :add_facet_paging_to_solr
+        :add_facet_paging_to_solr, :add_adv_search_clauses,
+        :add_additional_filters
       ]
     end
 
@@ -58,15 +59,69 @@ module Blacklight::Solr
       ##
       if search_field&.query_builder.present?
         add_search_field_query_builder_params(solr_parameters)
+      elsif search_field&.clause_params.present?
+        add_search_field_with_json_query_parameters(solr_parameters)
       elsif search_field&.solr_local_parameters.present?
         add_search_field_with_local_parameters(solr_parameters)
       elsif search_state.query_param.is_a? Hash
-        add_multifield_search_query(solr_parameters)
-      elsif blacklight_params[:q]
-        solr_parameters[:q] = search_state.query_param
+        add_additional_filters(solr_parameters, search_state.query_param)
+      elsif search_state.query_param
+        solr_parameters.append_query search_state.query_param
       end
     end
 
+    def add_additional_filters(solr_parameters, additional_filters = nil)
+      q = additional_filters || @additional_filters
+
+      return if q.blank?
+
+      if q.values.any?(&:blank?)
+        # if any field parameters are empty, exclude _all_ results
+        solr_parameters.append_query "{!lucene}NOT *:*"
+      else
+        composed_query = q.map do |field, values|
+          "#{field}:(#{Array(values).map { |x| solr_param_quote(x) }.join(' OR ')})"
+        end.join(" AND ")
+
+        solr_parameters.append_query "{!lucene}#{composed_query}"
+      end
+
+      solr_parameters[:defType] = 'lucene'
+      solr_parameters[:spellcheck] = 'false'
+    end
+
+    def add_search_field_with_json_query_parameters(solr_parameters)
+      bool_query = search_field.clause_params.transform_values { |v| v.merge(query: search_state.query_param) }
+
+      solr_parameters.append_boolean_query(:must, bool_query)
+    end
+
+    # Transform "clause" parameters into the Solr JSON Query DSL
+    def add_adv_search_clauses(solr_parameters)
+      return if search_state.clause_params.blank?
+
+      defaults = { must: [], must_not: [], should: [] }
+      default_op = blacklight_params[:op]&.to_sym || :must
+      solr_parameters[:mm] = 1 if default_op == :should && search_state.clause_params.values.any? { |clause| }
+
+      search_state.clause_params.each_value do |clause|
+        op, query = adv_search_clause(clause, default_op)
+        next unless defaults.key?(op)
+
+        solr_parameters.append_boolean_query(op, query)
+      end
+    end
+
+    # @return [Array] the first element is the query operator and the second is the value to add
+    def adv_search_clause(clause, default_op)
+      op = clause[:op]&.to_sym || default_op
+      field = (blacklight_config.search_fields || {})[clause[:field]] if clause[:field]
+
+      return unless field&.clause_params && clause[:query].present?
+
+      [op, field.clause_params.transform_values { |v| v.merge(query: clause[:query]) }]
+    end
+
     ##
     # Add any existing facet limits, stored in app-level HTTP query
     # as :f, to solr as appropriate :fq query.
@@ -80,18 +135,38 @@ module Blacklight::Solr
         if filter.config.filter_query_builder
           filter_query, subqueries = filter.config.filter_query_builder.call(self, filter, solr_parameters)
 
-          solr_parameters.append_filter_query(filter_query)
+          solr_parameters.append_filter_query(filter_query) if filter_query
           solr_parameters.merge!(subqueries) if subqueries
         else
           filter.values.reject(&:blank?).each do |value|
-            filter_query, subqueries = facet_value_to_fq_string(filter.config.key, value)
-            solr_parameters.append_filter_query(filter_query)
+            filter_query, subqueries = if value.is_a?(Array)
+                                         facet_inclusive_value_to_fq_string(filter.key, value.reject(&:blank?))
+                                       else
+                                         facet_value_to_fq_string(filter.config.key, value)
+                                       end
+
+            solr_parameters.append_filter_query filter_query
             solr_parameters.merge!(subqueries) if subqueries
           end
         end
       end
     end
 
+    def add_solr_facet_json_params(solr_parameters, field_name, facet, **additional_parameters)
+      solr_parameters[:json] ||= { facet: {} }
+      solr_parameters[:json][:facet] ||= {}
+
+      field_config = facet.json.respond_to?(:reverse_merge) ? facet.json : {}
+
+      field_config = field_config.reverse_merge(
+        type: 'terms',
+        field: facet.field,
+        limit: facet_limit_with_pagination(field_name)
+      ).merge(additional_parameters)
+
+      solr_parameters[:json][:facet][field_name] = field_config.select { |_k, v| v.present? }
+    end
+
     ##
     # Add appropriate Solr facetting directives in, including
     # taking account of our facet paging/'more'.  This is not
@@ -100,6 +175,11 @@ module Blacklight::Solr
       facet_fields_to_include_in_request.each do |field_name, facet|
         solr_parameters[:facet] ||= true
 
+        if facet.json
+          add_solr_facet_json_params(solr_parameters, field_name, facet)
+          next
+        end
+
         if facet.pivot
           solr_parameters.append_facet_pivot with_ex_local_param(facet.ex, facet.pivot.join(","))
         elsif facet.query
@@ -169,9 +249,7 @@ module Blacklight::Solr
 
       facet_config = blacklight_config.facet_fields[facet]
 
-      # Now override with our specific things for fetching facet values
-      facet_ex = facet_config.respond_to?(:ex) ? facet_config.ex : nil
-      solr_params[:"facet.field"] = with_ex_local_param(facet_ex, facet_config.field)
+      solr_params[:rows] = 0
 
       limit = if solr_params["facet.limit"]
                 solr_params["facet.limit"].to_i
@@ -184,13 +262,21 @@ module Blacklight::Solr
       prefix = search_state.facet_prefix
       offset = (page - 1) * limit
 
+      if facet_config.json
+        add_solr_facet_json_params(solr_parameters, facet, facet_config, limit: limit + 1, offset: offset, sort: sort, prefix: prefix)
+        return
+      end
+
+      # Now override with our specific things for fetching facet values
+      facet_ex = facet_config.respond_to?(:ex) ? facet_config.ex : nil
+      solr_params[:"facet.field"] = with_ex_local_param(facet_ex, facet_config.field)
+
       # Need to set as f.facet_field.facet.* to make sure we
       # override any field-specific default in the solr request handler.
       solr_params[:"f.#{facet_config.field}.facet.limit"] = limit + 1
       solr_params[:"f.#{facet_config.field}.facet.offset"] = offset
       solr_params[:"f.#{facet_config.field}.facet.sort"] = sort if sort
       solr_params[:"f.#{facet_config.field}.facet.prefix"] = prefix if prefix
-      solr_params[:rows] = 0
     end
 
     def with_ex_local_param(ex, value)
@@ -250,16 +336,17 @@ module Blacklight::Solr
 
     ##
     # Convert a facet/value pair into a solr fq parameter
-    def facet_value_to_fq_string(facet_field, value)
+    def facet_value_to_fq_string(facet_field, value, use_local_params: true)
       facet_config = blacklight_config.facet_fields[facet_field]
 
       solr_field = facet_config.field if facet_config && !facet_config.query
       solr_field ||= facet_field
 
       local_params = []
-      local_params << "tag=#{facet_config.tag}" if facet_config && facet_config.tag
 
-      prefix = "{!#{local_params.join(' ')}}" unless local_params.empty?
+      if use_local_params
+        local_params << "tag=#{facet_config.tag}" if facet_config && facet_config.tag
+      end
 
       if facet_config && facet_config.query
         if facet_config.query[value]
@@ -269,12 +356,34 @@ module Blacklight::Solr
           '-*:*'
         end
       elsif value.is_a?(Range)
+        prefix = "{!#{local_params.join(' ')}}" unless local_params.empty?
         "#{prefix}#{solr_field}:[#{value.first} TO #{value.last}]"
       else
         "{!term f=#{solr_field}#{(' ' + local_params.join(' ')) unless local_params.empty?}}#{convert_to_term_value(value)}"
       end
     end
 
+    def facet_inclusive_value_to_fq_string(facet_field, values)
+      return if values.blank?
+
+      return facet_value_to_fq_string(facet_field, values.first) if values.length == 1
+
+      facet_config = blacklight_config.facet_fields[facet_field]
+
+      local_params = []
+      local_params << "tag=#{facet_config.tag}" if facet_config && facet_config.tag
+
+      solr_filters = values.each_with_object({}).with_index do |(v, h), index|
+        h["f_inclusive.#{facet_field}.#{index}"] = facet_value_to_fq_string(facet_field, v, use_local_params: false)
+      end
+
+      filter_query = solr_filters.keys.map do |k|
+        "{!query v=$#{k}}"
+      end.join(' OR ')
+
+      ["{!lucene#{(' ' + local_params.join(' ')) unless local_params.empty?}}#{filter_query}", solr_filters]
+    end
+
     def convert_to_term_value(value)
       if value.is_a?(DateTime) || value.is_a?(Time)
         value.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -288,7 +397,7 @@ module Blacklight::Solr
     ##
     # The key to use to retrieve the grouped field to display
     def grouped_key_for_results
-      blacklight_config.index.group
+      blacklight_config.view_config(action_name: :index).group
     end
 
     def facet_fields_to_include_in_request
@@ -306,7 +415,7 @@ module Blacklight::Solr
     def add_search_field_query_builder_params(solr_parameters)
       q, additional_parameters = search_field.query_builder.call(self, search_field, solr_parameters)
 
-      solr_parameters[:q] = q
+      solr_parameters.append_query q
       solr_parameters.merge!(additional_parameters) if additional_parameters
     end
 
@@ -314,7 +423,7 @@ module Blacklight::Solr
       local_params = search_field.solr_local_parameters.map do |key, val|
         key.to_s + "=" + solr_param_quote(val, quote: "'")
       end.join(" ")
-      solr_parameters[:q] = "{!#{local_params}}#{search_state.query_param}"
+      solr_parameters.append_query "{!#{local_params}}#{search_state.query_param}"
 
       ##
       # Set Solr spellcheck.q to be original user-entered query, without
@@ -322,20 +431,5 @@ module Blacklight::Solr
       # params!
       solr_parameters["spellcheck.q"] ||= search_state.query_param
     end
-
-    def add_multifield_search_query(solr_parameters)
-      q = search_state.query_param
-      solr_parameters[:q] = if q.values.any?(&:blank?)
-                              # if any field parameters are empty, exclude _all_ results
-                              "{!lucene}NOT *:*"
-                            else
-                              "{!lucene}" + q.map do |field, values|
-                                "#{field}:(#{Array(values).map { |x| solr_param_quote(x) }.join(' OR ')})"
-                              end.join(" AND ")
-                            end
-
-      solr_parameters[:defType] = 'lucene'
-      solr_parameters[:spellcheck] = 'false'
-    end
   end
 end