supplejack_api 1

data/app/models/supplejack_api/concept.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  class Concept
+    include Support::Concept::Storable
+    include Support::Concept::Searchable
+    include ActiveModel::SerializerSupport
+
+    def self.build_context(fields)
+      context = {}
+      namespaces = []
+
+      fields.each do |field|
+        namespaces << ConceptSchema.model_fields[field].try(:namespace)
+      end
+
+      namespaces.compact.uniq.each do |namespace|
+        context[namespace] = ConceptSchema.namespaces[namespace].url
+        namespaced_fields = ConceptSchema.fields.select { |_key, field| field.namespace == namespace }
+        namespaced_fields.each do |name, field|
+          if fields.include?(name) && name != field.namespace
+            context[name] = "#{field.namespace}:#{field.namespace_field}"
+          end
+        end
+      end
+
+      # Manually build context for concept_id
+      context[:dcterms] = ConceptSchema.namespaces[:dcterms].url
+      context[:concept_id] = {}
+      context[:concept_id]['@id'] = 'dcterms:identifier'
+
+      fields.each do |field|
+        context[field] = {}
+        namespace = ConceptSchema.model_fields[field].try(:namespace)
+        context[field]['@id'] = "#{namespace}:#{field}"
+      end
+      context
+    end
+  end
+end

data/app/models/supplejack_api/concept_search.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  class ConceptSearch < Search
+    def initialize(options = {})
+      @options = options.dup
+      @options.reverse_merge!(
+        and: {},
+        or: {},
+        without: {},
+        page: 1,
+        per_page: 20,
+        sort: nil,
+        direction: 'desc',
+        fields: ConceptSchema.model_fields.keys.join(','),
+        debug: nil
+      )
+    end
+
+    # FIXME: make me smaller!
+    def search_builder
+      @search_builder ||= Sunspot.new_search(SupplejackApi::Concept) do
+        spellcheck collate: true, only_more_popular: true if options[:suggest]
+
+        options[:without].each do |name, values|
+          values = values.split(',')
+          values.each do |value|
+            without(name, to_proper_value(name, value))
+          end
+        end
+
+        if options[:geo_bbox]
+          coords = options[:geo_bbox].split(',').map(&:to_f)
+          with(:lat_lng).in_bounding_box([coords[2], coords[1]], [coords[0], coords[3]])
+        end
+
+        adjust_solr_params do |params|
+          if options[:solr_query].present?
+            params[:q] ||= ''
+            params['q.alt'] = options[:solr_query]
+            params[:defType] = 'dismax'
+          end
+        end
+
+        order_by(sort, direction) if options[:sort].present?
+
+        paginate page: page, per_page: per_page
+      end
+
+      @search_builder.build(&build_conditions)
+      @search_builder
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def query_fields
+      query_fields_list = super
+      query_fields_list += [:name, :label] if (query_fields_list && [:name, :label]).present?
+    end
+
+    def field_list
+      return @field_list if @field_list
+      model_fields = ConceptSchema.model_fields.dup
+      valid_fields = model_fields.keep_if { |_key, field| field.try(:store).nil? }
+
+      @field_list = options[:fields].split(',').map { |f| f.strip.tr(':', '_').to_sym }
+      @field_list.keep_if do |f|
+        valid_fields.include?(f)
+      end
+
+      @field_list
+    end
+
+    def group_list
+      return @group_list if @group_list
+      @group_list = options[:fields].split(',').map { |f| f.strip.to_sym }
+      @group_list.keep_if { |f| ConceptSchema.groups.keys.include?(f) }
+      @group_list
+    end
+  end
+end

data/app/models/supplejack_api/concerns/queryable_by_date.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Concerns
+    # Contains convenience methods for querying documents by their day field
+    # This concern assumes documents have the field 'day' of type +Date+
+    module QueryableByDate
+      extend ActiveSupport::Concern
+
+      included do
+        index date: 1
+      end
+
+      module ClassMethods
+        def created_on(date)
+          where(:date.gte => date.at_beginning_of_day, :date.lte => date.at_end_of_day)
+        end
+
+        def created_between(start_date, end_date)
+          where(:date.gte => start_date, :date.lte => end_date)
+        end
+      end
+    end
+  end
+end

data/app/models/supplejack_api/concerns/record.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+module SupplejackApi::Concerns::Record
+  extend ActiveSupport::Concern
+
+  included do
+    attr_accessor :next_record, :previous_record, :next_page, :previous_page
+    attr_accessor :should_index_flag
+
+    # Associations
+    embeds_many :fragments, cascade_callbacks: true, class_name: 'SupplejackApi::ApiRecord::RecordFragment'
+    embeds_one :merged_fragment, class_name: 'SupplejackApi::ApiRecord::RecordFragment'
+    has_and_belongs_to_many :concepts, class_name: 'SupplejackApi::Concept'
+
+    # From storable
+    store_in collection: 'records'
+    index(concept_ids: 1)
+    index({ record_id: 1 }, unique: true)
+    auto_increment :record_id, session: 'strong'
+
+    # Callbacks
+    before_save :merge_fragments
+
+    # Scopes
+    scope :active,          -> { where(status: 'active') }
+    scope :deleted,         -> { where(status: 'deleted') }
+    scope :suppressed,      -> { where(status: 'suppressed') }
+    scope :solr_rejected,   -> { where(status: 'solr_rejected') }
+
+    build_model_fields
+
+    def self.created_on(date)
+      where(:created_at.gte => date.at_beginning_of_day, :created_at.lte => date.at_end_of_day)
+    end
+
+    def self.find_multiple(ids)
+      return [] unless ids.try(:any?)
+      string_ids = ids.find_all { |id| id.to_s.length > 10 }
+      integer_ids = ids.find_all { |id| id.to_s.length <= 10 }
+
+      records = []
+      records += active.find(string_ids) if string_ids.present?
+      records += active.where(:record_id.in => integer_ids)
+      records = records.sort_by { |r| ids.find_index(r.record_id) || 100 }
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    # FIXME: make me smaller!
+    def find_next_and_previous_records(scope, options = {})
+      return unless options.try(:any?)
+
+      search = ::SupplejackApi::RecordSearch.new(options)
+      search.scope = scope
+
+      return nil unless search.valid?
+
+      # Find the index in the array for the current record
+      record_index = search.hits.find_index { |i| i.primary_key == id.to_s }
+      total_pages = (search.total.to_f / search.per_page).ceil
+
+      self.next_page = search.page
+      self.previous_page = search.page
+
+      return unless record_index
+
+      if record_index.zero?
+        unless search.page == 1
+          previous_page_search = ::SupplejackApi::RecordSearch.new(options.merge(page: search.page - 1))
+          previous_primary_key = previous_page_search.hits[-1].try(:primary_key)
+          self.previous_page = search.page - 1
+        end
+      else
+        previous_primary_key = search.hits[record_index - 1].try(:primary_key)
+      end
+
+      if previous_primary_key.present?
+        self.previous_record = SupplejackApi::Record.find(previous_primary_key).try(:record_id) rescue nil
+      end
+
+      if record_index == search.hits.size - 1
+        unless search.page >= total_pages
+          next_page_search = ::SupplejackApi::RecordSearch.new(options.merge(page: search.page + 1))
+          next_primary_key = next_page_search.hits[0].try(:primary_key)
+          self.next_page = search.page + 1
+        end
+      else
+        next_primary_key = search.hits[record_index + 1].try(:primary_key)
+      end
+
+      return unless next_primary_key.present?
+
+      self.next_record = ::SupplejackApi::Record.find(next_primary_key).try(:record_id) rescue nil
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def active?
+      status == 'active'
+    end
+
+    def should_index?
+      return should_index_flag unless should_index_flag.nil?
+      active?
+    end
+
+    def fragment_class
+      SupplejackApi::ApiRecord::RecordFragment
+    end
+  end
+end

data/app/models/supplejack_api/concerns/record_fragmentable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+module SupplejackApi::Concerns::RecordFragmentable
+  extend ActiveSupport::Concern
+
+  included do
+    embedded_in :record
+
+    delegate :record_id, to: :record
+
+    def self.schema_class
+      RecordSchema
+    end
+
+    build_mongoid_schema
+
+    def self.mutable_fields
+      @@mutable_fields ||= begin
+        immutable_fields = %w(_id _type source_id created_at updated_at)
+        mutable_fields = fields.keys - immutable_fields
+        Hash[mutable_fields.map { |name| [name, fields[name].type] }]
+      end.freeze
+    end
+  end
+end

data/app/models/supplejack_api/concerns/searchable.rb

@@ -0,0 +1,430 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/ModuleLength
+module SupplejackApi::Concerns::Searchable
+  extend ActiveSupport::Concern
+
+  included do
+    INTEGER_ATTRIBUTES ||= [:page, :per_page, :facets_per_page, :facets_page, :record_type].freeze
+
+    attr_accessor :options, :request_url, :scope, :solr_request_params, :errors, :warnings
+
+    class_attribute :max_values
+
+    self.max_values = {
+      page: 100_000,
+      per_page: 100,
+      facets_per_page: 150,
+      facets_page: 5000
+    }
+
+    def initialize(options = {})
+      @options = options.dup
+      @options.reverse_merge!(
+        facets: '',
+        and: {},
+        or: {},
+        without: {},
+        page: 1,
+        per_page: 20,
+        record_type: 0,
+        facets_per_page: 10,
+        facets_page: 1,
+        sort: nil,
+        direction: 'desc',
+        fields: 'default',
+        facet_query: {},
+        debug: nil
+      )
+    end
+
+    def self.model_class
+      to_s.gsub(/Search/, '').constantize
+    end
+
+    def self.schema_class
+      "#{model_class.to_s.demodulize}Schema".constantize
+    end
+
+    # The records that match the criteria within each role will be removed
+    # from the search results
+    #
+    def self.role_collection_restrictions(scope)
+      restrictions = []
+
+      if scope
+        role = scope.role.try(:to_sym)
+        if schema_class.roles[role].record_restrictions
+          restrictions = schema_class.roles[role].record_restrictions
+        end
+      end
+
+      restrictions
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/AbcSize
+    # FIXME: Make this method smaller, it's triple the max method length
+    def search_builder
+      search_model = self
+
+      @search_builder ||= Sunspot.new_search(search_model.class.model_class) do
+        with(:record_type, record_type) unless options[:record_type] == 'all'
+
+        search_model.facet_list.each do |facet_name|
+          facet(facet_name, limit: facets_per_page, offset: facets_offset)
+        end
+
+        spellcheck collate: true, only_more_popular: true if options[:suggest]
+
+        options[:without].each do |name, values|
+          values = values.split(',')
+          values.each do |value|
+            without(name, to_proper_value(name, value))
+          end
+        end
+
+        if options[:geo_bbox]
+          coords = options[:geo_bbox].split(',').map(&:to_f)
+          with(:lat_lng).in_bounding_box([coords[2], coords[1]], [coords[0], coords[3]])
+        end
+
+        adjust_solr_params do |params|
+          if options[:solr_query].present?
+            params[:q] ||= ''
+            params['q.alt'] = options[:solr_query]
+            params[:defType] = 'dismax'
+          end
+        end
+
+        # Facet Queries
+        #
+        # The facet query parameter should have the following format:
+        #
+        #   facet_query: {images: {"creator" => "all"}, headings: {"record_type" => 1}}
+        #
+        # - Each key in the top level hash will be the name of each facet row returned.
+        # - Each value in the top level hash is a hash similar with all the restrictions
+        #
+
+        if options[:facet_query].any?
+          facet(:counts) do
+            options[:facet_query].each_pair do |row_name, filters_hash|
+              row(row_name.to_s) do
+                filters_hash.each_pair do |filter, value|
+                  if value == 'all'
+                    without(filter.to_sym, nil)
+                  elsif filter =~ /-(.+)/
+                    without(Regexp.last_match(1).to_sym, to_proper_value(filter, value))
+                  else
+                    if value.is_a?(Array)
+                      with(filter.to_sym).all_of(value)
+                    else
+                      with(filter.to_sym, to_proper_value(filter, value))
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        order_by(sort, direction) if options[:sort].present?
+
+        search_model.class.role_collection_restrictions(search_model.scope).each do |field, values|
+          without(field.to_sym, values)
+        end
+
+        SupplejackApi::Source.suppressed.each do |source|
+          without(:source_id, source.source_id)
+        end
+
+        paginate page: page, per_page: per_page
+      end
+
+      @search_builder.build(&build_conditions)
+
+      @search_builder
+    end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength
+
+    # Return an array of valid facets
+    # It will remove any invalid facets in order to avoid Solr errors
+    #
+    def facet_list
+      return @facet_list if @facet_list
+
+      @facet_list = options[:facets].split(',').map { |f| f.strip.to_sym }
+      @facet_list.keep_if { |f| self.class.model_class.valid_facets.include?(f) }
+      @facet_list
+    end
+
+    def field_list
+      return @field_list if @field_list
+      valid_fields = self.class.schema_class.fields.keys.dup
+
+      @field_list = options[:fields].split(',').map { |f| f.strip.tr(':', '_').to_sym }
+      @field_list.delete_if do |f|
+        !valid_fields.include?(f)
+      end
+
+      @field_list
+    end
+
+    # Returns the facets part of the search results converted to a hash
+    #
+    # @return [Hash] facets in hash form, each facet is respresented as +facet_name => facet_values+
+    def facets_hash
+      facets = {}
+
+      self.facets.each do |facet|
+        rows = {}
+        facet.rows.each do |row|
+          rows[row.value] = row.count
+        end
+
+        facets.merge!(facet.name => rows)
+      end
+
+      facets
+    end
+
+    # Returns all valid groups of fields
+    # The groups are extracted from the "fields" parameter
+    #
+    def group_list
+      return @group_list if @group_list
+      @group_list = options[:fields].split(',').map { |f| f.strip.to_sym }
+      @group_list.keep_if { |f| self.class.model_class.valid_groups.include?(f) }
+      @group_list
+    end
+
+    def query_fields
+      query_field_list = nil
+
+      if options[:query_fields].is_a?(String)
+        query_field_list = options[:query_fields].split(',').map(&:strip).map(&:to_sym)
+      elsif options[:query_fields].is_a?(Array)
+        query_field_list = options[:query_fields].map(&:to_sym)
+      end
+
+      return nil if query_field_list.try(:empty?)
+      query_field_list
+    end
+
+    def extract_range(value)
+      if value =~ /^\[(\d+)\sTO\s(\d+)\]$/
+        Regexp.last_match(1).to_i..Regexp.last_match(2).to_i
+      else
+        value.to_i.positive? ? value.to_i : value.strip
+      end
+    end
+
+    def to_proper_value(_name, value)
+      return false if value == 'false'
+      return true if value == 'true'
+      return nil if %w(nil null).include?(value)
+
+      value = value.strip if value.is_a?(String)
+      value
+    end
+
+    # Downcase all queries before sending to SOLR, except queries
+    # which have specific lucene syntax.
+    #
+    def text
+      @text = options[:text]
+      if @text.present? && !@text.match(/:\"/)
+        @text.downcase!
+        @text.gsub!(/ and | or | not /, &:upcase)
+      end
+      @text
+    end
+
+    def offset
+      (page * per_page) - per_page
+    end
+
+    def facets_offset
+      offset = (facets_page * facets_per_page) - facets_per_page
+      offset.negative? ? 0 : offset
+    end
+
+    def valid?
+      self.errors ||= []
+      self.warnings ||= []
+      self.class.max_values.each do |attribute, max_value|
+        max_value = self.class.max_values[attribute]
+        if @options[attribute].to_i > max_value
+          self.warnings << "The #{attribute} parameter can not exceed #{max_value}"
+        end
+      end
+
+      solr_search_object
+      self.errors.empty?
+    end
+
+    def records
+      solr_search_object.results
+    end
+
+    def jsonp
+      @options[:jsonp].present? ? @options[:jsonp] : nil
+    end
+
+    # It's currently required to make the active_model_serializers gem to work with XML
+    # The XML Serialization is handled by the respective serializer
+    #
+    def to_xml; end
+
+    # IMPORTANT !!!!
+    #
+    # Try to make this a bit prettier
+    #
+    INTEGER_ATTRIBUTES.each do |method|
+      define_method(method) do
+        value = @options[:"#{method.to_sym}"].to_i
+        value = [value, self.class.max_values[method]].min if self.class.max_values[method]
+        value
+      end
+    end
+
+    def sort
+      value = @options[:sort].to_sym
+
+      begin
+        field = Sunspot::Setup.for(self.class.model_class).field(value)
+        return value
+      rescue Sunspot::UnrecognizedFieldError => e
+        return 'score'
+      end
+    end
+
+    def direction
+      if %w(asc desc).include?(@options[:direction])
+        @options[:direction].to_sym
+      else
+        :desc
+      end
+    end
+
+    def solr_search_object
+      return @solr_search_object if @solr_search_object
+      @solr_search_object = execute_solr_search
+
+      if options[:debug] == 'true' && @solr_search_object.respond_to?(:query)
+        self.solr_request_params = @solr_search_object.query.to_params
+      end
+
+      @solr_search_object
+    end
+
+    def solr_error_message(exception)
+      {
+        title: "#{exception.response[:status]} #{RSolr::Error::Http::STATUS_CODES[exception.response[:status].to_i]}",
+        body: exception.send(:parse_solr_error_response, exception.response[:body])
+      }
+    end
+
+    # rubocop:disable Style/MethodMissing
+    def method_missing(symbol, *args)
+      return nil unless solr_search_object.respond_to?(:hits)
+      solr_search_object.send(symbol, *args)
+    end
+    # rubocop:enable Style/MethodMissing
+
+    def execute_solr_search
+      search = search_builder
+
+      search.build do
+        keywords text, fields: query_fields
+      end
+
+      execute_solr_search_and_handle_errors(search)
+    end
+
+    def execute_solr_search_and_handle_errors(search)
+      self.errors ||= []
+      sunspot = search.execute
+    rescue RSolr::Error::Http => e
+      self.errors << solr_error_message(e)
+      Rails.logger.info e.message
+      sunspot = {}
+    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET => e
+      self.errors << 'Solr is temporarily unavailable please try again in a few seconds.'
+      Rails.logger.info e.message
+      sunspot = {}
+    ensure
+      return sunspot
+    end
+
+    private
+
+    # Generates the :and and :or conditions for a search object
+    #
+    def build_conditions
+      proc do
+        { and: options[:and], or: options[:or] }.each do |operator, value|
+          Utils.call_block(self, &recurse_conditions(operator, value))
+        end
+      end
+    end
+
+    # Detects when the key is a operator (:and, :or) and calls itself
+    # recursively until it finds facets defined in Sunspot.
+    #
+    def recurse_conditions(key, conditions, current_operator = :and)
+      proc do
+        case key.to_sym
+        when :and
+          all_of do
+            conditions.each do |filter, value|
+              Utils.call_block(self, &recurse_conditions(filter, value, :and))
+            end
+          end
+        when :or
+          any_of do
+            conditions.each do |filter, value|
+              Utils.call_block(self, &recurse_conditions(filter, value, :or))
+            end
+          end
+        else
+          Utils.call_block(self, &filter_values(key, conditions, current_operator))
+        end
+      end
+    end
+
+    # Generates a single condition. It can take a operator to
+    # determine how the values within the filter are going to be
+    # joined.
+    #
+    def filter_values(key, conditions, current_operator = :and)
+      proc do
+        if conditions.is_a? Hash
+          operator, values = conditions.first
+        else
+          operator = current_operator
+          values = conditions
+        end
+
+        if values.is_a?(Array)
+          case operator.to_sym
+          when :or
+            with(key).any_of(values)
+          when :and
+            with(key).all_of(values)
+          else
+            raise Exception, 'Expected operator (:and, :or)'
+          end
+        else
+          if values =~ /(.+)\*$/
+            with(key).starting_with(Regexp.last_match(1))
+          else
+            with(key, to_proper_value(key, values))
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ModuleLength