supplejack_api 1

data/app/models/supplejack_api/support/harvestable.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Support
+    module Harvestable
+      extend ActiveSupport::Concern
+
+      def create_or_update_fragment(attributes)
+        if fragment = find_fragment(attributes['source_id'])
+          fragment.clear_attributes
+        elsif fragments.count.zero?
+          fragment = primary_fragment
+        else
+          fragment = fragments.build
+        end
+
+        fragment.update_from_harvest(attributes)
+      end
+
+      def update_from_harvest(attributes = {})
+        attributes = attributes.try(:symbolize_keys) || {}
+
+        attributes[:status] ||= 'active'
+
+        self.class.fields.each do |name, _field|
+          if attributes.key?(name.to_sym)
+            value = Array(attributes[name.to_sym]).first
+            send("#{name}=", value)
+          end
+        end
+
+        primary_fragment.update_from_harvest(attributes)
+        self.updated_at = Time.now
+      end
+
+      def update_from_harvest!(attributes = {})
+        update_from_harvest(attributes)
+        save
+      end
+
+      def set_status(required_fragments)
+        missing_fragments = Array(required_fragments) - fragments.map(&:source_id)
+        self.status = missing_fragments.empty? ? 'active' : 'partial'
+      end
+
+      # Sets all the Record attributes to nil except the internal ones
+      # that shouldn't be removed (_id, _type, etc..)
+      #
+      def clear_attributes
+        self[:source_url] = nil
+        primary_fragment.clear_attributes
+      end
+
+      def unset_null_fields
+        raw_json = raw_attributes || {}
+        unset_hash = {}
+        raw_json.each do |key, value|
+          unset_hash.merge!(key => true) if value.nil?
+        end
+        if raw_json['fragments'].present?
+          raw_json['fragments'].each_with_index do |fragment, index|
+            next if fragment.nil?
+            fragment.each do |key, value|
+              unset_hash.merge!("fragments.#{index}.#{key}" => true) if value.nil?
+            end
+          end
+        end
+        collection.find(atomic_selector).update('$unset' => unset_hash) if unset_hash.any?
+      end
+
+      module ClassMethods
+        def find_or_initialize_by_identifier(attributes)
+          attributes = attributes.symbolize_keys
+          identifier = attributes.delete(:internal_identifier)
+          identifier = identifier.first if identifier.is_a?(Array)
+          find_or_initialize_by(internal_identifier: identifier)
+        end
+
+        def flush_old_records(source_id, job_id)
+          where(
+            :'fragments.source_id' => source_id,
+            :'fragments.job_id'.ne => job_id,
+            :status.in => %w(active supressed)
+          ).update_all(status: 'deleted', updated_at: Time.now)
+
+          cursor = deleted.where('fragments.source_id': source_id)
+          total = cursor.count
+          start = 0
+          chunk_size = 10_000
+          while start < total
+            records = cursor.limit(chunk_size).skip(start)
+            Sunspot.remove(records)
+            start += chunk_size
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/supplejack_api/support/searchable.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Support
+    module Searchable
+      extend ActiveSupport::Concern
+
+      included do
+        include Sunspot::Mongoid
+
+        searchable if: :should_index? do
+          string :internal_identifier
+          integer :record_type
+
+          string :source_id do
+            primary_fragment.source_id
+          end
+
+          build_sunspot_schema(self)
+        end
+      end
+
+      SUNSPOT_TYPE_NAMES = {
+        string: :string,
+        integer: :integer,
+        datetime: :time,
+        boolean: :boolean,
+        latlon: :latlon
+      }.freeze
+
+      module ClassMethods
+        def schema_class
+          "#{to_s.demodulize}Schema".constantize
+        end
+
+        def custom_find(id, scope = nil, options = {})
+          options ||= {}
+          class_scope = unscoped
+          class_scope = class_scope.active unless options.delete(:status) == :all
+          column = "#{name.demodulize.downcase}_id"
+
+          if id.to_s =~ /^\d+$/
+            data = class_scope.where(column => id).first
+          elsif id.to_s =~ /^[0-9a-f]{24}$/i
+            data = class_scope.find(id)
+          end
+
+          raise Mongoid::Errors::DocumentNotFound.new(self, [id], [id]) unless data
+
+          begin
+            data.find_next_and_previous_records(scope, options) if options.any?
+          rescue Sunspot::UnrecognizedFieldError,
+                 RSolr::Error::Http, Timeout::Error,
+                 Errno::ECONNREFUSED,
+                 Errno::ECONNRESET => e
+
+            Rails.logger.error e.inspect
+          end
+
+          data
+        end
+
+        def build_sunspot_schema(builder)
+          schema_class.fields.each do |_name, field|
+            options = {}
+            search_as = field.search_as || []
+
+            value_block = nil
+            if field.search_value.present?
+              value_block = proc do
+                field.search_value.call(self)
+              end
+            end
+
+            options[:as] = field.solr_name if field.solr_name.present?
+
+            if search_as.include? :filter
+              filter_options = {}
+              filter_options[:multiple] = true if field.multi_value.present?
+              type = SUNSPOT_TYPE_NAMES[field.type]
+
+              builder.public_send(type, field.name, options.merge(filter_options), &value_block)
+            end
+
+            if search_as.include? :fulltext
+              options[:boost] = field.search_boost if field.search_boost.present?
+              builder.text field.name, options, &value_block
+            end
+          end
+        end
+
+        def valid_facets
+          facets = []
+
+          schema_class.fields.each do |name, field|
+            search_as = field.search_as || []
+            facets << name.to_sym if search_as.include? :filter
+          end
+
+          facets
+        end
+
+        def valid_groups
+          schema_class.groups.keys
+        end
+      end
+    end
+  end
+end

data/app/models/supplejack_api/support/status_logger.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Support
+    module StatusLogger
+      def self.logger
+        logfile = File.open("#{Rails.root}/log/status.log", 'a')
+        logfile.sync = true
+        @logger ||= Logger.new(logfile)
+        @logger.formatter = proc do |_severity, datetime, _progname, msg|
+          "#{datetime.to_formatted_s(:db)} #{msg}\n"
+        end
+        @logger
+      end
+    end
+  end
+end

data/app/models/supplejack_api/support/storable.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Support
+    module Storable
+      extend ActiveSupport::Concern
+
+      included do
+        include Mongoid::Document
+        include Mongoid::Timestamps
+        include Mongoid::Attributes::Dynamic
+
+        # Both of these fields are required in SJ API Core
+        # No need to configure in *Schema
+        field :internal_identifier,         type: String
+        field :status,                      type: String
+        field :record_type,                 type: Integer,      default: 0
+
+        index status: 1
+        index({ internal_identifier: 1 }, unique: true, drop_dups: true)
+        index record_type: 1
+        index({ record_id: 1 }, unique: true)
+
+        def self.build_model_fields
+          if RecordSchema.model_fields.present?
+            RecordSchema.model_fields.each do |name, index|
+              # Set the field
+              field name.to_sym, index.field_options if !!index.field_options
+
+              # Set the index
+              index_fields = !!index.index_fields ? index.index_fields : {}
+              index_options = !!index.index_options ? index.index_options : {}
+              index index_fields, index_options unless index_fields.empty?
+
+              # Set the validation
+              validates name.to_sym, index.validation if !!index.validation
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/supplejack_api/support/validation_logger.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  module Support
+    module ValidationLogger
+      def self.logger
+        logfile = File.open("#{Rails.root}/log/validation.log", 'a')
+        logfile.sync = true  # automatically flush data to file
+        @logger ||= Logger.new(logfile)
+        @logger.formatter = proc do |_severity, datetime, _progname, msg|
+          "#{datetime.to_formatted_s(:db)}: #{msg}\n"
+        end
+        @logger
+      end
+    end
+  end
+end

data/app/models/supplejack_api/usage_metrics.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  class UsageMetrics
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    include SupplejackApi::Concerns::QueryableByDate
+
+    store_in collection: 'usage_metrics'
+
+    field :record_field_value,         type: String
+    field :searches,         	         type: Integer, default: 0
+    field :gets,             	         type: Integer, default: 0
+    field :user_set_views,   	         type: Integer, default: 0
+    field :total_views,                type: Integer, default: 0
+    field :records_added_to_user_sets, type: Integer, default: 0
+    field :date,                       type: Date
+  end
+end

data/app/models/supplejack_api/user.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+# TODO: Remove custom_search stuff to slim the class?
+# rubocop:disable Metrics/ClassLength
+module SupplejackApi
+  class User
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    include ActiveModel::ForbiddenAttributesProtection
+    include Sortable::Query
+
+    store_in collection: 'users'
+
+    # Include default devise modules. Others available are:
+    # :token_authenticatable, :encryptable, :confirmable, :lockable, :timeoutable and :omniauthable
+    devise :trackable, :database_authenticatable, :token_authenticatable
+
+    # Database authenticatable
+    # field :email,              type: String, null: false
+    # field :encrypted_password, type: String, null: false
+    field :email,               type: String
+    field :encrypted_password,  type: String
+    field :name,                type: String
+    field :username,            type: String
+
+    index authentication_token: 1
+
+    # Trackable
+    field :sign_in_count,      type: Integer
+    field :current_sign_in_at, type: Time
+    field :last_sign_in_at,    type: Time
+    field :current_sign_in_ip, type: String
+    field :last_sign_in_ip,    type: String
+
+    # Token authenticatable
+    field :authentication_token, type: String
+
+    field :daily_requests,    type: Integer,  default: 0
+    field :monthly_requests,  type: Integer,  default: 0
+    field :max_requests,      type: Integer,  default: 10_000
+    field :role,              type: String,   default: 'developer'
+
+    field :daily_activity,        type: Hash
+    field :daily_activity_stored, type: Boolean, default: true
+    index daily_activity_stored: 1
+
+    has_many :user_activities, class_name: 'SupplejackApi::UserActivity'
+
+    scope :with_daily_activity, -> { where(daily_activity_stored: false) }
+
+    alias api_key authentication_token
+
+    before_save :ensure_authentication_token
+
+    has_many :user_sets, dependent: :destroy, autosave: true, class_name: 'SupplejackApi::UserSet' do
+      def custom_find(id)
+        user_set = if BSON::ObjectId.legal?(id.to_s)
+                     find(id) rescue nil
+                   else
+                     where(url: id).first
+                   end
+      end
+    end
+
+    def sets=(attrs_array)
+      return false unless attrs_array.try(:any?)
+      attrs_array.each { |attrs| user_sets.build(attrs) }
+    end
+
+    def name
+      name = self[:name]
+      name.present? ? name : username
+    end
+
+    def updated_today?
+      updated_at > Time.now.beginning_of_day
+    end
+
+    def check_daily_requests
+      if updated_today?
+        increment_daily_requests
+      else
+        self.daily_requests = 1
+      end
+
+      if daily_requests == (max_requests * 0.9).floor
+        SupplejackApi::RequestLimitMailer.at90percent(self).deliver
+        SupplejackApi::RequestLimitMailer.at90percent_admin(self).deliver
+      elsif daily_requests == max_requests
+        SupplejackApi::RequestLimitMailer.at100percent(self).deliver
+        SupplejackApi::RequestLimitMailer.at100percent_admin(self).deliver
+      end
+    end
+
+    def increment_daily_requests
+      self.daily_requests ||= 0
+      self.daily_requests += 1
+    end
+
+    def update_tracked_fields(request)
+      old_current = current_sign_in_at
+      new_current = Time.now.utc
+      self.last_sign_in_at     = old_current || new_current
+      self.current_sign_in_at  = new_current
+
+      old_current = current_sign_in_ip
+      new_current = request.ip
+      self.last_sign_in_ip     = old_current || new_current
+      self.current_sign_in_ip  = new_current
+
+      self.sign_in_count ||= 0
+      self.sign_in_count += 1
+    end
+
+    def update_daily_activity(request)
+      # Get the controller name and strips out the `supplejack_api/` namespace
+      controller = request.params[:controller].to_s.gsub(%r{ supplejack_api/ }, '')
+      action = request.params[:action].to_s
+
+      if controller == 'records' && action == 'index'
+        controller = 'search'
+        action = 'records'
+      # TODO: Custom Search is deprecated. Evaluate for removal
+      elsif controller == 'custom_searches' && action == 'records'
+        controller = 'search'
+        action = 'custom_search'
+      elsif controller == 'set_items'
+        controller = 'user_sets'
+        action = "#{action}_item"
+      end
+
+      self.daily_activity ||= {}
+      self.daily_activity[controller] ||= {}
+
+      current_value = self.daily_activity[controller][action].to_i
+      self.daily_activity[controller][action] = current_value + 1
+
+      self.daily_activity_stored = false
+    end
+
+    def reset_daily_activity
+      self.daily_requests = 0
+      self.daily_activity = nil
+      self.daily_activity_stored = true
+    end
+
+    def over_limit?
+      updated_today? && daily_requests > max_requests
+    end
+
+    def calculate_last_30_days_requests
+      count = 0
+      user_activities.gt(created_at: Time.now - 30.days).each { |activity| count += activity.total.to_i }
+      self.monthly_requests = count
+    end
+
+    def requests_per_day(days = 30)
+      today = Time.zone.now.to_date
+      user_activities = self.user_activities.gt(created_at: today - days.days).asc(:created_at).to_a
+
+      requests = []
+      date = today - days.days
+      while date < today
+        date += 1.days
+        requests << (user_activities.find { |ua| ua.created_at.to_date == date }.try(:total) || 0)
+      end
+      requests
+    end
+
+    def name_or_user
+      name.present? ? split_on_at_symbol(name) : split_on_at_symbol(username)
+    end
+
+    def split_on_at_symbol(value)
+      if value.present? && value.include?('@')
+        value.split('@').first
+      else
+        value
+      end
+    end
+
+    def self.find_by_api_key(api_key)
+      where(authentication_token: api_key).first
+    end
+
+    def self.custom_find(id)
+      user = if id.to_s.length == 24
+               find(id)
+             else
+               find_by_api_key(id)
+             end
+
+      raise Mongoid::Errors::DocumentNotFound.new(self, id, id) unless user
+      user
+    end
+
+    def admin?
+      RecordSchema.roles[role.to_sym].try(:admin)
+    end
+
+    def can_change_featured_sets?
+      admin?
+    end
+  end
+end