solr_lite 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d21097855d584cb46f8ee7e0ac4cd25daf0208e8
+  data.tar.gz: 3b420f8645190a2ea69d23077bc7634cfa71dc74
+SHA512:
+  metadata.gz: a510c01a80f26d5ef2d9a1eb71dd5ee55fbe8045598a05952bc4ad8c0e5ec36ef45f3c96b07b28cf47e3f68b3c56f9c19b3836c561e0a17a5542256541bbe4cf
+  data.tar.gz: 9a906c6991457e1c2cc91b4d43bd22be1a12392ed5628f639a5f133145e99df4133f93fe5aabc039a5e3718be4519bbc4cf15ce0a106db9a770c5afacd4f69f1

data/lib/explain_entry.rb

@@ -0,0 +1,50 @@
+module SolrLite
+  class ExplainEntry
+    attr_accessor :key, :score, :matches, :top_matches, :text
+
+    def initialize(key, text)
+      @key = key
+      # get rid of this one extraneous linebreak that Solr includes
+      @text = text.gsub("\n), product of:", ", product of:")
+      @matches = get_matches(@text)
+      @score = @matches.first.split(" ").first
+      @top_matches = get_top_matches(@matches)
+    end
+
+    private
+      # For a given entry, returns an array with the lines that include match
+      # information (notice that we also include line with the coord() value)
+      def get_matches(text)
+        lines = text.split("\n")
+        lines.select {|l| l.include?("(MATCH)") || l.include?("coord(")}
+      end
+
+      # A subset of the matches that includes only the match that was picked
+      # i.e. the one with the higher score.
+      # (notice that we also include line with the coord() value)
+      def get_top_matches(matches)
+        top = []
+        token_score = nil
+        matches.each do |line|
+          if line.include?("max of")
+            token_score = line.strip.split().first
+          else
+            if token_score == nil
+              # If we don't have a score to match this line is probably a
+              # "product of" or "sum of" marker. Include it.
+              top << line
+            else
+              if line.strip.start_with?(token_score)
+                top << line
+              elsif line.include?("coord(")
+                top << line
+              else
+                # Ignore it, must be a line with a match that was not picked.
+              end
+            end
+          end
+        end
+        top
+      end
+  end
+end

data/lib/explainer.rb

@@ -0,0 +1,34 @@
+require "explain_entry.rb"
+module SolrLite
+  class Explainer
+    attr_accessor :entries
+
+    # solr_response_hash a Solr HTTP response parsed via JSON.parse()
+    def initialize(solr_reponse_hash)
+      @explain = solr_reponse_hash.fetch("debug", {}).fetch("explain", [])
+      @entries = @explain.map do |ex|
+        key = ex[0]
+        text = ex[1]
+        ExplainEntry.new(ex[0], ex[1])
+      end
+    end
+
+    # solr_response (string) is the Solr HTTP response from a query
+    def self.from_response(solr_response)
+      hash = JSON.parse(solr_response)
+      Explainer.new(hash)
+    end
+
+    # Raw string with the explain information for each entry
+    def text()
+      text = ""
+      @entries.each do |entry|
+        text += "-- #{entry.key} {\r\n"
+        text += "#{entry.text}\r\n"
+        text += "}\r\n"
+        text += "\r\n"
+      end
+      text
+    end
+  end
+end

data/lib/facet_field.rb

@@ -0,0 +1,60 @@
+module SolrLite
+  class FacetField
+
+    class FacetValue
+      attr_accessor :text, :count, :remove_url, :add_url
+      def initialize(text = "", count = 0, remove_url = nil)
+        @text = text
+        @count = count
+        @remove_url = remove_url
+        @add_url = nil
+      end
+    end
+
+    attr_accessor :name, :title, :values
+    def initialize(name, display_value)
+      @name = name # field name in Solr
+      @title = display_value
+      @values = []
+    end
+
+    def to_qs(text)
+      "#{@name}|#{CGI.escape(text)}"
+    end
+
+    def add_value(text, count)
+      @values << FacetValue.new(text, count)
+    end
+
+    def value_count(text)
+      v = @values.find {|v| v.text == text}
+      return 0 if v == nil
+      v.count
+    end
+
+    def set_remove_url_for(value, url)
+      @values.each do |v|
+        if v.text == value
+          v.remove_url = url
+        end
+      end
+    end
+
+    def set_add_url_for(value, url)
+      @values.each do |v|
+        if v.text == value
+          v.add_url = url
+        end
+      end
+    end
+
+    def set_urls_for(value, add_url, remove_url)
+      @values.each do |v|
+        if v.text == value
+          v.add_url = add_url
+          v.remove_url = remove_url
+        end
+      end
+    end
+  end
+end

data/lib/filter_query.rb

@@ -0,0 +1,68 @@
+require "cgi"
+
+module SolrLite
+  # Represents an "fq" in Solr. Field is the field to filter by
+  # and value the value to filter by. In a Solr query are represented
+  # as "fq=field:value"
+  class FilterQuery
+    attr_accessor :field, :value, :solr_value, :qs_value, :form_value
+    attr_accessor :title, :remove_url
+
+    def initialize(field, values)
+      @field = field
+      @value = values.join("|")
+      @solr_value = to_solr_fq_value(field, values)
+      @qs_value = "#{field}"
+      values.each do |v|
+        @qs_value += "|#{CGI.escape(v)}"    # URL friendly (no : or quotes)
+      end
+      @form_value = "#{field}|#{@value}"    # HTML Form friendly (no encoding, the form auto-encodes on POST)
+      @title = field                        # default to field name
+      @remove_url = nil
+    end
+
+    # qs is assumed to be the value taken from the query string
+    # in the form `field|value` or `field|value1|valueN`.
+    #
+    # Sometimes(*) the string comes URL encoded, for example:
+    #     `field|hello`
+    #     `field|hello%20world`
+    # CGI.unespace handles these cases nicer than URL.decode
+    #
+    # (*) Values coming from HTML forms submitted via HTTP POST tend
+    # to be encoded slighly different than value submitted via
+    # HTTP GET requests.
+    def self.from_query_string(qs)
+      tokens = CGI.unescape(qs).split("|")
+      return nil if tokens.count < 2
+      field = ""
+      values = []
+      tokens.each_with_index do |token, i|
+        if i == 0
+          field = token
+        else
+          values << token
+        end
+      end
+      FilterQuery.new(field, values)
+    end
+
+    private
+      # Creates a filter query (fq) string as needed by Solr from
+      # an array of values. Handles single and multi-value gracefully.
+      # For single-value it returns "(field:value)". For multi-value
+      # it returns "(field:value1) OR (field:value2)"
+      def to_solr_fq_value(field, values)
+        solr_value = ""
+        values.each_with_index do |v, i|
+          solr_value += '(' + field + ':"' + v + '")'
+          lastValue = (i == (values.length-1))
+          if !lastValue
+            solr_value += " OR "
+          end
+        end
+        # Very important to escape the : otherwise URL.parse throws an error in Linux
+        CGI.escape(solr_value)
+      end
+  end # class
+end # module

data/lib/response.rb

@@ -0,0 +1,114 @@
+require "facet_field.rb"
+require "explainer.rb"
+require "spellcheck.rb"
+module SolrLite
+  class Response
+    attr_accessor :items, :solr_response
+
+    def initialize(solr_response, params)
+      @solr_response = solr_response
+      @params = params
+      @explainer = nil
+      set_facet_values()
+
+      # This value can be set by the client if we want to use a custom
+      # representation of solr_docs
+      @items = []
+    end
+
+    def ok?
+      return true if status == 0
+      return true if status >= 200 && status <= 299
+      false
+    end
+
+    def status
+      return -1 if @solr_response["responseHeader"] == nil
+      @solr_response["responseHeader"]["status"]
+    end
+
+    def error_msg
+      return "" if @solr_response["error"] == nil
+      return "" if @solr_response["error"]["msg"] == nil
+      @solr_response["error"]["msg"]
+    end
+
+    # Total number documents found in solr
+    # usually larger than solr_docs.count
+    def num_found
+      @solr_response["response"]["numFound"]
+    rescue
+      0
+    end
+
+    def num_pages
+      return 0 if page_size == 0
+      pages = (num_found / page_size).to_i
+      pages += 1 if (num_found % page_size) != 0
+      pages
+    end
+
+    def page_size
+      @solr_response["responseHeader"]["params"]["rows"].to_i
+    rescue
+      0
+    end
+
+    # Start position for retrieval (used for pagination)
+    def start
+      @solr_response["response"]["start"].to_i
+    rescue
+      0
+    end
+
+    def end
+      [start + page_size, num_found].min
+    end
+
+    def page
+      return 1 if page_size == 0 # fail safe
+      (start / page_size) + 1
+    end
+
+    # Raw solr_docs
+    def solr_docs
+      @solr_response["response"]["docs"]
+    end
+
+    def facets
+      @params.facets
+    end
+
+    def set_facet_values()
+      return if @solr_response["facet_counts"] == nil
+      solr_facets = @solr_response["facet_counts"]["facet_fields"]
+      solr_facets.each do |solr_facet|
+        # solr_facet is an array with two elements, e.g.
+        # ["record_type", ["PEOPLE", 32, "ORGANIZATION", 4]]
+        #
+        # the first element has the field for the facet (record_type)
+        # the second element is an array with of value/count pairs (PEOPLE/32, ORG/4)
+        field_name = solr_facet[0]
+        facet_field = @params.facet_for_field(field_name)
+        values = solr_facet[1]
+        pairs = values.count/2
+        for pair in (1..pairs)
+          index = (pair-1) * 2
+          text = values[index]
+          count = values[index+1]
+          facet_field.add_value(text, count)
+        end
+      end
+      # TODO: make sure we sort the FacetField.VALUES descending by count
+    end
+
+    def explainer()
+      @explainer ||= SolrLite::Explainer.new(@solr_response)
+    end
+
+    def spellcheck()
+      @spellcheck ||= SolrLite::Spellcheck.new(@solr_response)
+    end
+
+  end # class
+end # module

data/lib/search_params.rb

@@ -0,0 +1,167 @@
+require "filter_query.rb"
+require "facet_field.rb"
+module SolrLite
+  class SearchParams
+    attr_accessor :q, :fq, :facets, :page, :page_size, :fl, :sort, :facet_limit
+
+    DEFAULT_PAGE_SIZE = 20
+
+    def initialize(q = "", fq = [], facets = [])
+      @q = q
+      @fq = fq          # array of FilterQuery
+      @facets = facets  # array of FacetField
+      @page = 1
+      @page_size = DEFAULT_PAGE_SIZE
+      @fl = nil
+      @sort = ""
+      @facet_limit = nil
+    end
+
+    def facet_for_field(field)
+      @facets.find {|f| f.name == field}
+    end
+
+    def set_facet_remove_url(field, value, url)
+      facet = facet_for_field(field)
+      if facet != nil
+        facet.set_remove_url_for(value, url)
+      end
+    end
+
+    def start_row()
+      (@page - 1) * @page_size
+    end
+
+    def star_row=(start)
+      # recalculate the page
+      if @page_size == 0
+        @page = 0
+      else
+        @page = (start / @page_size) + 1
+      end
+      nil
+    end
+
+    # Returns the string that we need render on the Browser to execute
+    # a search with the current parameters.
+    #
+    # facet_to_ignore: a FilterQuery object with a value to ignore when
+    #   creating the query string.
+    # q_override: a string with a Solr query to use instead of the current q value
+    def to_user_query_string(facet_to_ignore = nil, q_override = nil)
+      qs = ""
+      q_value = q_override != nil ? q_override : @q
+      if q_value != "" && @q != "*"
+        qs += "&q=#{@q}"
+      end
+      @fq.each do |filter|
+        if facet_to_ignore != nil && filter.solr_value == facet_to_ignore.solr_value
+          # don't add this to the query string
+        else
+          qs += "&fq=#{filter.qs_value}"
+        end
+      end
+      qs += "&rows=#{@page_size}" if @page_size != DEFAULT_PAGE_SIZE
+      qs += "&page=#{@page}" if @page != 1
+      # Don't surface this to the UI for now
+      # (since we don't let the user change the sorting)
+      # qs += "&sort=#{@sort}" if sort != ""
+      qs
+    end
+
+    def to_user_query_string_no_q()
+      to_user_query_string(nil, '')
+    end
+
+    # Returns the string that we need to pass Solr to execute a search
+    # with the current parameters.
+    def to_solr_query_string(extra_fqs = [])
+      qs = ""
+      if @q != ""
+        qs += "&q=#{@q}"
+      end
+      @fq.each do |filter|
+        qs += "&fq=#{filter.solr_value}"
+      end
+      extra_fqs.each do |filter|
+        qs += "&fq=#{filter.solr_value}"
+      end
+      qs += "&rows=#{@page_size}"
+      qs += "&start=#{start_row()}"
+      if sort != ""
+        qs += "&sort=#{CGI.escape(@sort)}"
+      end
+      if @facets.count > 0
+        qs += "&facet=on"
+        @facets.each do |f|
+          qs += "&facet.field=#{f.name}"
+          qs += "&f.#{f.name}.facet.mincount=1"
+          if @facet_limit != nil
+            qs += "&f.#{f.name}.facet.limit=#{@facet_limit}"
+          end
+        end
+      end
+      qs
+    end
+
+    # Returns an array of values that can be added to an HTML form
+    # to represent the current search parameters. Notice that we do
+    # NOT include the `q` parameter there is typically an explicit
+    # HTML form value for it on the form.
+    def to_form_values()
+      values = []
+
+      # We create an individual fq_n HTML form value for each
+      # fq value because Rails does not like the same value on the form.
+      @fq.each_with_index do |filter, i|
+        values << {name: "fq_#{i}", value: filter.form_value}
+      end
+
+      values << {name: "rows", value: @page_size} if @page_size != DEFAULT_PAGE_SIZE
+      values << {name: "page", value: @page} if @page != 1
+      # Don't surface this to the UI for now
+      # (since we don't let the user change the sorting)
+      # values << {name: "sort", value: @sort} if sort != ""
+      values
+    end
+
+    def to_s()
+      "q=#{@q}\nfq=#{@fq}"
+    end
+
+    def self.from_query_string(qs, facet_fields = [])
+      params = SearchParams.new
+      params.facets = facet_fields
+      tokens = qs.split("&")
+      tokens.each do |token|
+        values = token.split("=")
+        name = values[0]
+        value = values[1]
+        fq = nil
+        next if value == nil || value.empty?
+        case
+        when name == "q"
+          params.q = value
+        when name == "rows"
+          params.page_size = value.to_i
+        when name == "page"
+          params.page = value.to_i
+        when name == "fq" || name.start_with?("fq_")
+          # Query string contains fq when _we_ build the query string, for
+          # example as the user clicks on different facets on the UI.
+          # A query string can have multiple fq values.
+          #
+          # Query string contains fq_n when _Rails_ pushes HTML FORM values to
+          # the query string. Rails does not like duplicate values in forms
+          # and therefore we force them to be different by appending a number
+          # to them (fq_1, f1_2, ...)
+          fq = FilterQuery.from_query_string(value)
+          if fq != nil
+            params.fq << fq
+          end
+        end
+      end
+      params
+    end
+  end
+end

data/lib/solr.rb

@@ -0,0 +1,174 @@
+require "net/http"
+require "time"
+require "json"
+require "search_params.rb"
+require "response.rb"
+module SolrLite
+  class Solr
+    # Creates an instance of the Solr class.
+    # Parameters:
+    #   solr_url: string with the URL to Solr ("http://localhost:8983/solr/bibdata")
+    #   logger: an instance of Rails::logger if using Rails.
+    #       Could also be SolrLite::Logger which defaults to the console.
+    #       Or nil to omit logging.
+    def initialize(solr_url, logger = nil)
+      raise "No solr_url was indicated" if solr_url == nil
+      @solr_url = solr_url
+      @logger = logger
+    end
+
+    # Fetches a Solr document by id.
+    # Parameters:
+    #   id: ID of the document to fetch.
+    #   q_field: Query field (defaults to "q")
+    #   fl: list of fields to fetch (defaults to "*")
+    #
+    # Returns a hash with the document information or nil if no document was found.
+    # Raises an exception if more than one document was found.
+    def get(id, q_field = "q", fl = "*")
+      query_string = "#{q_field}=id%3A#{id}"     # %3A => :
+      query_string += "&fl=#{fl}"
+      query_string += "&wt=json&indent=on"
+      url = "#{@solr_url}/select?#{query_string}"
+      solr_response = Response.new(http_get(url), nil)
+      if solr_response.num_found > 1
+        raise "More than one record found for id #{id}"
+      end
+      solr_response.solr_docs.first
+    end
+
+    # Issues a search request to Solr.
+    # Parameters:
+    #   params: an instance of SolrParams.
+    #   extra_fqs: array of FilterQuery objects. This is used to
+    #     add filters to the search that we don't want to allow the
+    #     user to override.
+    #   qf: Used to override the server's qf value.
+    #   mm: Used to override the server's mm value.
+    #   debug: true to include debugQuery info in the response. (defaults to false)
+    #
+    # Returns an instance of SolrLite::Response
+    def search(params, extra_fqs = [], qf = nil, mm = nil, debug = false)
+      if params.fl != nil
+        query_string = "fl=#{params.fl.join(",")}"
+      else
+        query_string = "" # use Solr defaults
+      end
+
+      query_string += "&wt=json&indent=on"
+      query_string += "&" + params.to_solr_query_string(extra_fqs)
+      query_string += "&q.op=AND"
+
+      if qf != nil
+        query_string += "&qf=#{CGI.escape(qf)}"
+      end
+
+      if mm != nil
+        query_string += "&mm=#{CGI.escape(mm)}"
+      end
+
+      if debug
+        query_string += "&debugQuery=true"
+      end
+
+      url = "#{@solr_url}/select?#{query_string}"
+      http_response = http_get(url)
+      response = Response.new(http_response, params)
+      response
+    end
+
+    # shortcut for search
+    def search_text(terms)
+      params = SearchParams.new(terms)
+      search(params)
+    end
+
+    def start_row(page, page_size)
+      (page - 1) * page_size
+    end
+
+    def update(json)
+      url = @solr_url + "/update?commit=true"
+      solr_response = http_post_json(url, json)
+      solr_response
+    end
+
+    def delete_by_id(id)
+      # Use XML format here because that's the only way I could get
+      # the delete to recognize ids with a colon (e.g. bdr:123).
+      # Using JSON caused the Solr parser to choke.
+      #
+      # Notice that they payload is XML but the response is JSON (wt=json)
+      url = @solr_url + "/update?commit=true&wt=json"
+      payload = "<delete><id>#{id}</id></delete>"
+      http_response = http_post(url, payload, "text/xml")
+      solr_response = Response.new(JSON.parse(http_response), nil)
+      solr_response
+    end
+
+    def delete_by_query(query)
+      url = @solr_url + "/update?commit=true"
+      payload = '{ "delete" : { "query" : "' + query + '" } }'
+      solr_response = http_post_json(url, payload)
+      solr_response
+    end
+
+    def delete_all!()
+      delete_by_query("*:*")
+    end
+
+    private
+      def http_post_json(url, payload)
+        content_type = "application/json"
+        http_response = http_post(url, payload, content_type)
+        Response.new(JSON.parse(http_response), nil)
+      end
+
+      def http_post(url, payload, content_type)
+        start = Time.now
+        log_msg("Solr HTTP POST #{url}")
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        if url.start_with?("https://")
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request["Content-Type"] = content_type
+        request.body = payload
+        response = http.request(request)
+        log_elapsed(start, "Solr HTTP POST")
+        response.body
+      end
+
+      def http_get(url)
+        start = Time.now
+        log_msg("Solr HTTP GET #{url}")
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        if url.start_with?("https://")
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request["Content-Type"] = "application/json"
+        response = http.request(request)
+        log_elapsed(start, "Solr HTTP GET")
+        JSON.parse(response.body)
+      end
+
+      def elapsed_ms(start)
+        ((Time.now - start) * 1000).to_i
+      end
+
+      def log_elapsed(start, msg)
+        log_msg("#{msg} took #{elapsed_ms(start)} ms")
+      end
+
+      def log_msg(msg)
+        if @logger != nil
+          @logger.info(msg)
+        end
+      end
+  end
+end

data/lib/solr_lite.rb

@@ -0,0 +1 @@
+require "solr.rb"

data/lib/spellcheck.rb

@@ -0,0 +1,58 @@
+module SolrLite
+  class Spellcheck
+    def initialize(solr_reponse_hash)
+      @spellcheck = solr_reponse_hash.fetch("spellcheck", {})
+    end
+
+    def suggestions()
+      @suggestions ||= @spellcheck.fetch("suggestions",[])
+    end
+
+    def collations()
+      @collations ||= begin
+        collations = @spellcheck.fetch("collations",nil)
+        if collations != nil
+          if collations.kind_of?(Array)
+            # We must be in Solr6, use the collation information as-is
+          else
+            # uh-oh...
+            []
+          end
+        else
+          # We must be on Solr4, mimic the structure of the Solr6 results
+          # which is an array in the form:
+          #
+          #   ["collation", {"collationQuery": "wordA"}, "collation", {"collationQuery": "wordB"}, ...]
+          #
+          # As a reference, the structure in Solr4 is slightly different in that
+          # the collationQuery information is in an array within an array:
+          #
+          #   ["collation", ["collationQuery", "wordA"], "collation"["collationQuery", "wordB"], ...]
+          #
+          collations = []
+          suggestions = suggestions()
+          suggestions.each_with_index do |x, i|
+            if x == "collation"
+              collationQuery = suggestions[i+1]
+              word = collationQuery[1]
+              collations << "collation"
+              collations << {"collationQuery" => word}
+            end
+          end
+        end
+        collations
+      end
+    end
+
+    # def spellcheck_correctly_spelled()
+    #   @spellcheck.fetch("correctlySpelled", true)
+    # end
+
+    def top_collation_query()
+      colls = collations()
+      return nil if colls.length < 2
+      top_collation = colls[1] || {}
+      top_collation.fetch("collationQuery", nil)
+    end
+  end
+end

metadata

@@ -0,0 +1,53 @@
+--- !ruby/object:Gem::Specification
+name: solr_lite
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Hector Correa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-23 00:00:00.000000000 Z
+dependencies: []
+description: A lightweight gem to connect to Solr and run queries. Requires no extra
+  dependencies.
+email: hector_correa@brown.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/explain_entry.rb
+- lib/explainer.rb
+- lib/facet_field.rb
+- lib/filter_query.rb
+- lib/response.rb
+- lib/search_params.rb
+- lib/solr.rb
+- lib/solr_lite.rb
+- lib/spellcheck.rb
+homepage: https://github.com/Brown-University-Library/solr_lite
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A lightweight gem to connect to Solr and run queries
+test_files: []