quick_search-core 0.0.1.test

data/app/controllers/quick_search/appstats_controller.rb

@@ -0,0 +1,210 @@
+module QuickSearch
+  class AppstatsController < ApplicationController
+    include Auth
+
+    before_action :auth, :start_date, :end_date, :days_in_sample
+
+    def index
+      @page_title = 'Search Statistics'
+      search_click_ratio
+    end
+
+    def clicks_overview
+      @page_title = 'Clicks Overview'
+      module_clicks
+      result_types_clicks
+      typeahead_clicks
+    end
+
+    def top_searches
+      @page_title = 'Top Searches'
+      searches_count
+      searches = Search.where(page: 'index').where(date_range)
+      queries = []
+      searches.each do |search|
+        queries << search["query"].downcase
+      end
+      @query_counts = Hash.new(0)
+      queries.each do |query|
+        @query_counts[query] += 1
+      end
+      @query_counts = @query_counts.sort_by {|k,v| v}.reverse[0..199]
+    end
+
+    def top_spot
+      @page_title = params[:ga_top_spot_module]
+      top_spot_reporting(params[:ga_top_spot_module])
+    end
+
+    def detail
+      start_date
+      end_date
+      @category = params[:ga_category]
+      if params[:ga_scope] == 'action'
+        @action = params[:ga_action]
+        @page_title = "#{@category} (#{URI.decode(@action)})"
+        top_spot_detail(@category, @action)
+        render "top_spot_detail"
+      elsif params[:ga_scope] == 'category'
+        @page_title = "#{@category}"
+        module_click_detail(@category)
+        render "module_click_detail"
+      end
+    end
+
+    def realtime
+    end
+
+    private
+
+    def module_clicks
+      @module_clicks = Event.where(date_range).where(excluded_categories).group(:category).order("count_category DESC").count(:category)
+      @module_click_total = calculate_total_clicks(@module_clicks)
+    end
+
+    def typeahead_clicks
+      searches_count
+      @typeahead_clicks = Event.where(date_range).where("category LIKE 'typeahead-%'").group(:category).order("count_category DESC").count(:category)
+      @typeahead_clicks_total = calculate_total_clicks(@typeahead_clicks)
+    end
+
+    def result_types_clicks
+      @result_types_clicks = Event.where(date_range).where(:category => "result-types").group(:action).order("count_action DESC").count(:action)
+      @result_types_click_total = calculate_total_clicks(@result_types_clicks)
+    end
+
+    def calculate_total_clicks(clicks)
+      click_total = 0
+      clicks.each do |module_name, click_count|
+        click_total += click_count
+      end
+
+      click_total
+    end
+
+    def searches_count
+      @searches_count = Search.where(page: 'index').where(date_range).count
+    end
+
+    def search_click_ratio
+      searches_count
+      @clicks_count = Event.where(excluded_categories).where(date_range).count
+      @click_serve_ratio = (@clicks_count / @searches_count.to_f)*100
+    end
+
+    def top_spot_reporting(top_spot_module)
+      reports = {"best-bets-regular" => {:category_title => "Best Bets Regular"},
+                 "best-bets-algorithmic-journal" => {:category_title => "Journal Algorithmic Best Bets"},
+                 "best-bets-algorithmic-database" => {:category_title => "Database Algorithmic Best Bets"},
+                 "best-bets-course-tools" => {:category_title => "Course Tools Best Bets"},
+                 "spelling-suggestion" => {:category_title => "Spelling Suggestions"}
+               }
+      @top_spot_reporting = []
+      top_spot_report = base_query(top_spot_module)
+      @top_spot_reporting << {:category => top_spot_module, :category_title => reports[top_spot_module][:category_title], :report => top_spot_report}
+      # reports.each do |key, report|
+      #   top_spot_report = base_query(key)
+      #   @top_spot_reporting << {:category => key, :category_title => report[:category_title], :report => top_spot_report}
+      # end
+    end
+
+    def module_click_detail(category)
+       modules_clicks = Event.where(:category => "#{category}").where(date_range).group(:action).order('count_category DESC').count(:category)
+       total_clicks = modules_clicks.values.sum
+       @modules_clicks_report = []
+       modules_clicks.each do |module_clicks|
+        @modules_clicks_report << {:module => module_clicks[0], :clicks => module_clicks[1], :percent => (module_clicks[1]/total_clicks.to_f)*100}
+      end
+    end
+
+    def top_spot_detail(category, action)
+      # Use a join here?
+      action = URI.decode(action)
+      serves = Event.where(:category => "#{category}-serve", :action => "#{action}").where(date_range).group(:label).order("count_label DESC").count(:label)
+      clicks = Event.where(:category => "#{category}", :action => "#{action}").where(date_range).group(:label).order("count_label DESC").count(:label)
+
+      @top_spot_detail = {}
+
+
+      serves.each do |label|
+        serve_count = label[1].nil? ? 0:label[1]
+        click_count = clicks[label[0].downcase].nil? ? 0:clicks[label[0].downcase]
+        @top_spot_detail[label[0]] = [serve_count, click_count, (click_count/serve_count.to_f)*100]
+      end
+    end
+
+    def base_query(category)
+      serves = Event.where(date_range).where(:category => "#{category}-serve").group(:action).order("count_category DESC").count(:category)
+      clicks = Event.where(date_range).where(:category => "#{category}").group(:action).count(:category)
+
+      result = []
+      serves.each do |data, count|
+        unless clicks[data].nil?
+          click_count = clicks[data]
+        else
+          click_count = 0
+        end
+
+        row = {"action" => data,
+          "servecount" => count,
+          "clickcount" => click_count,
+          "click_serve_ratio" => (click_count/count)*100 }
+        result << row
+      end
+
+      result
+    end
+
+    def start_date
+      if params[:start_date]
+        # If there's a specified start date, use it
+        @start_date = convert_to_time(params[:start_date])
+      else
+        # otherwise use 6 months ago as a default, or the earliest event date if logs are younger
+        first_date = Event.first[:created_at]
+        start = Time.current - 6.months
+
+        if start < first_date
+          @start_date = first_date
+        else
+          @start_date = start
+        end
+      end
+
+    end
+
+    def end_date
+      if params[:end_date]
+        # use end date, if specified
+        @end_date = convert_to_time(params[:end_date])
+      else
+        # otherwise use the current date as default
+        @end_date = Time.current
+      end
+    end
+
+    def convert_to_time(date_input)
+      year = date_input[:year]
+      month = date_input[:month]
+      day = date_input[:day]
+
+      Time.new(year, month, day)
+    end
+
+    def days_in_sample
+      @days_in_sample = ((@end_date - @start_date) / (24*60*60)).round
+      if @days_in_sample < 1
+        @days_in_sample = 1
+      end
+    end
+
+    def excluded_categories
+      "category NOT LIKE \"%serve\" AND category <> \"common-searches\" AND category <> \"result-types\"AND category NOT LIKE \"typeahead-%\""
+    end
+
+    def date_range
+      { :created_at => start_date.beginning_of_day..end_date.end_of_day }
+    end
+
+  end
+end

data/app/controllers/quick_search/opensearch_controller.rb

@@ -0,0 +1,9 @@
+module QuickSearch
+  class OpensearchController < ApplicationController
+
+    def opensearch
+      response.headers['Content-Type'] = 'application/opensearchdescription+xml; charset=utf-8'
+    end
+
+  end
+end

data/app/controllers/quick_search/pages_controller.rb

@@ -0,0 +1,18 @@
+module QuickSearch
+  class PagesController < ApplicationController
+
+    include Auth
+
+    before_action :auth, only: [:realtime]
+
+    def home
+    end
+
+    def about
+    end
+
+    def realtime
+    end
+
+  end
+end

data/app/controllers/quick_search/search_controller.rb

@@ -0,0 +1,250 @@
+module QuickSearch
+  class SearchController < ApplicationController
+    include QuickSearch::SearcherConcern
+    include QuickSearch::DoiTrap
+    include QuickSearch::OnCampus
+    include QuickSearch::QueryParser
+    include QuickSearch::EncodeUtf8
+    include QuickSearch::QueryFilter
+    include QuickSearch::SearcherConfig
+
+    require 'benchmark_logger'
+
+    before_filter :doi_trap, :log_query
+    after_filter :realtime_message, only: [:index]
+    protect_from_forgery except: :log_event
+
+    def index
+      loaded_searches
+      @common_searches = common_searches
+      http_search
+    end
+
+    # TODO: throw error if required files not in place
+    def single_searcher
+      searcher_name = params[:searcher_name]
+
+      searcher_cfg = searcher_config(searcher_name)
+      if searcher_cfg and searcher_cfg.has_key? 'loaded_searches'
+        additional_services = Array.new(searcher_cfg['loaded_searches'])
+      else
+        additional_services = []
+      end
+      loaded_searches(additional_services)
+      @common_searches = searcher_cfg['common_searches'] || []
+      #TODO: maybe a default template for single-searcher searches?
+      http_search(searcher_name, "quick_search/search/#{searcher_name}_search")
+    end
+
+    # The following searches for individual sections of the page.
+    # This allows us to do client-side requests in cases where the original server-side
+    # request times out or otherwise fails.
+    def xhr_search
+      endpoint = params[:endpoint]
+
+      if params[:template] == 'with_paging'
+        template = 'xhr_response_with_paging'
+      else
+        template = 'xhr_response'
+      end
+
+      @query = params_q_scrubbed
+      @page = page
+      @per_page = per_page(endpoint)
+      @offset = offset(@page,@per_page)
+
+      http_client = HTTPClient.new
+      update_searcher_timeout(http_client, endpoint, true)
+
+      benchmark "%s xhr #{endpoint}" % CGI.escape(@query.to_str) do
+
+        klass = "QuickSearch::#{endpoint.camelize}Searcher".constantize
+        searcher = klass.new(http_client,
+                             extracted_query(params_q_scrubbed),
+                             @per_page,
+                             @offset,
+                             @page,
+                             on_campus?(ip),
+                             extracted_scope(params_q_scrubbed))
+        searcher.search
+
+        searcher_partials = {}
+        searcher_cfg = searcher_config(endpoint)
+        unless searcher_cfg.blank?
+          services = searcher_cfg['services'].blank? ? [] : searcher_cfg['services']
+        else
+          services = []
+        end
+        services << endpoint
+
+        respond_to do |format|
+
+          format.html {
+            services.each do |service|
+              service_template = render_to_string(
+                :partial => "quick_search/search/#{template}",
+                :layout => false,
+                :locals => { module_display_name: t("#{endpoint}_search.display_name"),
+                             searcher: searcher,
+                             search: '',
+                             service_name: service
+                            })
+              searcher_partials[service] = service_template
+            end
+            render :json => searcher_partials
+          }
+
+          format.json {
+
+            # prevents openstruct object from results being nested inside tables
+            # See: http://stackoverflow.com/questions/7835047/collecting-hashes-into-openstruct-creates-table-entry
+            result_list = []
+            searcher.results.each do |result|
+              result_list << result.to_h
+            end
+
+            render :json => { :endpoint => endpoint,
+                              :per_page => @per_page.to_s,
+                              :page => @page.to_s,
+                              :total => searcher.total,
+                              :results => result_list
+            }
+          }
+        end
+      end
+    end
+
+    def log_search
+      if params[:query].present? && params[:page].present?
+        Search.create(query: params[:query], page: params[:page])
+        render :nothing => true, :status => 200, :content_type => 'text/html'
+      else
+        render :nothing => true, :status => 500, :content_type => 'text/html'
+      end
+    end
+
+    def log_event
+      if params[:category].present? && params[:event_action].present? && params[:label].present?
+        Event.create(category: params[:category], action: params[:event_action], label: params[:label][0..250])
+        if params[:callback].present?
+          render :nothing => true, :status => 200, :content_type => 'text/javascript'
+        else
+          render :nothing => true, :status => 200, :content_type => 'text/html'
+        end
+      else
+        render :nothing => true, :status => 500, :content_type => 'text/html'
+      end
+    end
+
+    private
+
+    def http_search(endpoint = 'defaults', page_to_render = :index)
+      @ip = request.remote_ip
+
+      @search_form_placeholder = I18n.t "#{endpoint}_search.search_form_placeholder"
+      @page_title = I18n.t "#{endpoint}_search.display_name"
+      @module_callout = I18n.t "#{endpoint}_search.module_callout"
+
+      if search_in_params?
+        @query = params_q_scrubbed
+        @search_in_params = true
+        search_all_in_threads(endpoint)
+        #log_search(@query, page_to_render)
+        render page_to_render
+      else
+        @search_in_params = false
+        render '/quick_search/pages/home'
+      end
+    end
+
+    def page
+      if page_in_params?
+        page = params[:page].to_i
+      else
+        page = 1
+      end
+      page
+    end
+    helper_method :page
+
+    def per_page(endpoint)
+      searcher_cfg = searcher_config(endpoint)
+      if params[:per_page]
+        per_page = params[:per_page].to_i
+      elsif params[:template] == 'with_paging'
+        if searcher_cfg and searcher_cfg.has_key? 'with_paging'
+          per_page = searcher_cfg['with_paging']['per_page']
+        else
+          per_page = 10
+        end
+      else
+        per_page = QuickSearch::Engine::APP_CONFIG['per_page']
+      end
+
+      if per_page > QuickSearch::Engine::APP_CONFIG['max_per_page']
+        per_page = QuickSearch::Engine::APP_CONFIG['max_per_page']
+      end
+
+      per_page
+    end
+
+    def offset(page, per_page)
+      (page * per_page) - per_page
+    end
+
+    def page_in_params?
+      params[:page] && !params[:page].blank?
+    end
+
+    def search_in_params?
+      params_q_scrubbed && !params_q_scrubbed.blank?
+    end
+    helper_method :search_in_params?
+
+    def common_searches
+      QuickSearch::Engine::APP_CONFIG['common_searches']
+    end
+
+    def loaded_searches(additional_services=[])
+      @search_services_for_display = []
+      @extracted_query = extracted_query(params_q_scrubbed)
+      search_services = additional_services + Array.new(QuickSearch::Engine::APP_CONFIG['loaded_searches'])
+
+      search_services.each do |search_service|
+        if search_in_params?
+          @search_services_for_display << {'name' => search_service['name'], 'link'=> search_service['query'] + extracted_query(params_q_scrubbed)}
+        else
+          @search_services_for_display << {'name' => search_service['name'], 'link'=> search_service['landing_page']}
+        end
+      end
+    end
+
+    def realtime_message
+      if base_url = QuickSearch::Engine::APP_CONFIG['realtime_url']
+        begin
+          client = HTTPClient.new
+          body = {q: params_q_scrubbed}
+          url = File.join(base_url, "/message/quicksearch-#{Rails.env}")
+          res = client.post(url, body)
+        rescue
+        end
+      end
+    end
+
+    def benchmark(message)
+      result = nil
+      ms = Benchmark.ms { result = yield }
+      BenchmarkLogger.info '%s (%.1fms)' % [ message, ms ]
+      result
+    end
+
+    def log_query
+      if search_in_params?
+        @log_query = filter_query(params_q_scrubbed)
+      else
+        @log_query = ""
+      end
+    end
+
+  end
+end

data/app/helpers/quick_search/application_helper.rb

@@ -0,0 +1,68 @@
+module QuickSearch::ApplicationHelper
+  include QuickSearch::EncodeUtf8
+
+  def application_name
+    if action_name == "website"
+      I18n.t "website_search.title_name"
+    else
+      I18n.t "defaults_search.title_name"
+    end
+  end
+
+  def organization_name
+    I18n.t 'organization_name'
+  end
+
+  def sanitized_query
+    params_q_scrubbed.nil? || params_q_scrubbed == "" ? "" : sanitize(params_q_scrubbed)
+  end
+
+  def display_query
+    params_q_scrubbed.nil? || params_q_scrubbed == "" ? "" : truncate(params_q_scrubbed, length: 40, separator: ' ')
+  end
+
+  def log_query
+    params_q_scrubbed.nil? || params_q_scrubbed == "" ? "" : truncate(sanitize(params_q_scrubbed), length: 100, separator: ' ')
+  end
+
+  def title(page_title = nil)
+    query = params_q_scrubbed.nil? || params_q_scrubbed == "" ? "" : truncate(params_q_scrubbed, length: 40, separator: ' ', escape: false)
+    page_title ||= ''
+    page_title << "#{query} |" unless query.blank?
+    page_title << " #{application_name} |" unless application_name.blank?
+    page_title << " #{organization_name}" unless organization_name.blank?
+
+    content_for :title, page_title
+  end
+
+  # FIXME this should use en definitions
+  def pages(page)
+    format_pages ||= ''
+    if page.count('-') > 0
+      format_pages << "pp.&nbsp;"
+    else
+      format_pages << "p.&nbsp;"
+    end
+    format_pages << page
+  end
+
+  def render_module(searcher, service_name, partial_name = '/quick_search/search/module', per_page = 3)
+    render partial: partial_name , locals: { module_display_name: t("#{service_name}_search.display_name"), searcher: searcher, search: '', service_name: service_name, per_page: per_page }
+  end
+
+  def get_best_bets_data(best_bets_searcher)
+    best_bets = {}
+    best_bets[:id] = best_bets_searcher.results.first.id
+    best_bets[:title] = best_bets_searcher.results.first.title
+    best_bets[:link] = best_bets_searcher.results.first.link
+    best_bets[:description] = best_bets_searcher.results.first.description
+    best_bets[:best_bets_type] = best_bets_searcher.results.first.best_bets_type
+
+    best_bets
+  end
+
+ def body_class
+    [controller_name, action_name].join('-')
+  end
+
+end

data/app/helpers/quick_search/appstats_helper.rb

@@ -0,0 +1,2 @@
+module QuickSearch::AppstatsHelper
+end

data/app/helpers/quick_search/opensearch_helper.rb

@@ -0,0 +1,2 @@
+module QuickSearch::OpensearchHelper
+end

data/app/helpers/quick_search/pages_helper.rb

@@ -0,0 +1,2 @@
+module QuickSearch::PagesHelper
+end

data/app/helpers/quick_search/search_helper.rb

@@ -0,0 +1,2 @@
+module QuickSearch::SearchHelper
+end

data/app/models/event.rb

@@ -0,0 +1,2 @@
+class Event < ActiveRecord::Base
+end

data/app/models/search.rb

@@ -0,0 +1,11 @@
+class Search < ActiveRecord::Base
+  
+  def self.get_most_frequent_searches
+    frequent_searches = []
+    Search.all(:select => "query, count(query) as query_count", :group => "query", :order => "query_count DESC", :limit => 7).each do |row|
+      frequent_searches << row[:query]
+    end
+      frequent_searches
+  end
+
+end

data/app/searchers/quick_search/best_bets_searcher.rb

@@ -0,0 +1,102 @@
+module QuickSearch
+  class BestBetsSearcher < QuickSearch::Searcher
+
+    def search
+      search_best_bets_index
+    end
+
+    def results
+      if results_list
+        results_list
+      else
+        @results_list = []
+        unless @response.blank?
+          @results_list << @response
+        end
+        @results_list
+      end
+    end
+
+    def total
+      results.size
+    end
+
+    private
+
+    # Regular Best Bets
+
+    def search_best_bets_index
+      response = @http.get(query_url)
+      parsed_response = JSON.parse(response.body)
+      if parsed_response['response']['numFound'].to_s == '0'
+        return nil
+      else
+        resp = parsed_response['response']['docs'][0]
+        result = OpenStruct.new
+        result.title = title(resp)
+        result.link = link(resp)
+        result.id = id(resp)
+        result.description = description(resp)
+        result.best_bets_type = 'best-bets-regular'
+
+        @response = result
+      end
+    end
+
+    def base_url
+      QuickSearch::Engine::APP_CONFIG['best_bets']['solr_url']
+    end
+
+    def query_url
+      if http_request_queries['not_escaped'].split.size < 3
+       # use non-analyzed keyword field with phrased search
+        params = {
+          'q' => %Q(keywords: "#{query_quotes_stripped}")
+        }
+      else
+        # use DisMax query parser for longer queries
+        params = {
+          'defType' => 'dismax',
+          'q' => query_quotes_stripped
+        }
+        if http_request_queries['not_escaped'].split.size < 6
+          params['mm'] = '4'
+        else
+          params['mm'] = '6'
+        end
+      end
+
+      # return results in json
+      params['wt'] = 'json'
+      base_url + params.to_query
+    end
+
+    def title(value)
+      value['title']
+    end
+
+    def link(value)
+      # TODO: should we make all URLS in the best_bets index absolute? currently some begin with a /
+      if value['url'].start_with?('/')
+        QuickSearch::Engine::APP_CONFIG['website']['website_base_url'] + value['url']
+      else
+        value['url']
+      end
+    end
+
+    def id(value)
+      value['id']
+    end
+
+    def description(value)
+      value['description']
+    end
+
+    # General Methods
+
+    def query_quotes_stripped
+      http_request_queries['not_escaped'].gsub('"', '')
+    end
+
+  end
+end