qa_server 7.5.1 → 7.9.0

data/app/presenters/qa_server/monitor_status/history_up_down_presenter.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+# This presenter class provides historical testing data needed by the view that monitors status of authorities.
+module QaServer::MonitorStatus
+  class HistoryUpDownPresenter
+    attr_reader :historical_up_down_data
+
+    # @param parent [QaServer::MonitorStatusPresenter] parent presenter
+    # @param historical_up_down_data [Hash<Array>] recent connection status of queries (typically last 30 days)
+    # @example historical_up_down_data
+    #   { 'AGROVOC' = [
+    #       :FULLY_UP,   # 0 - today
+    #       :MOSTLY_UP,  # 1 - yesterday
+    #       :MOSTLY_UP,  # 2 - two days ago
+    #       :FULLY_UP,   # 3 - three days ago
+    #       :DOWN,       # 4 - four days ago
+    #       ...          # etc.
+    #     ],
+    #     'CERL' = [ ... ]
+    #   }
+    def initialize(parent:, historical_up_down_data:)
+      @parent = parent
+      @historical_up_down_data = historical_up_down_data
+    end
+
+    # Return the last date of data represented in the history graph and data table
+    # @return [ActiveSupport::TimeWithZone] date time stamp
+    def up_down_start
+      QaServer::TimeService.pretty_date(up_down_end_dt - 29.days)
+    end
+
+    def up_down_end
+      QaServer::TimeService.pretty_date(up_down_end_dt)
+    end
+
+    def up_down_end_dt
+      @parent.last_updated_dt
+    end
+
+    # @param status [Symbol] :fully_up, :mostly_up, :timeouts, :barely_up, :down
+    # @param day [Integer] retrieve the status for this day
+    # @return [String] name of the css class for the status
+    def historical_up_down_status_class(status, day) # rubocop:disable Metrics/CyclomaticComplexity
+      case status[day]
+      when :no_date then 'connection-no-date'
+      when :fully_up then 'connection-fully-up'
+      when :mostly_up then 'connection-mostly-up'
+      when :timeouts then 'connection-timeouts'
+      when :barely_up then 'connection-barely-up'
+      when :down then 'connection-down'
+      end
+    end
+
+    # @return [Boolean] true if historical datatable should be visible; otherwise false
+    def display_historical_up_down?
+      QaServer.config.display_historical_datatable? && @historical_up_down_data.present?
+    end
+  end
+end

data/app/presenters/qa_server/monitor_status_presenter.rb

@@ -5,12 +5,14 @@ module QaServer
     extend Forwardable
 
     # @param current_summary [ScenarioRunSummary] summary status of the latest run of test scenarios
-    # @param current_data [Array<Hash>] current set of failures for the latest test run, if any
+    # @param current_failure_data [Array<Hash>] current set of failures for the latest test run, if any
     # @param historical_summary_data [Array<Hash>] summary of past failuring runs per authority to drive chart
+    # @param historical_up_down_data [Hash<Array>] status of queries for the last 30 days
     # @param performance_data [Hash<Hash>] performance datatable data
-    def initialize(current_summary:, current_failure_data:, historical_summary_data:, performance_data:)
+    def initialize(current_summary:, current_failure_data:, historical_summary_data:, historical_up_down_data:, performance_data:)
       @current_status_presenter = QaServer::MonitorStatus::CurrentStatusPresenter.new(parent: self, current_summary: current_summary, current_failure_data: current_failure_data)
       @history_presenter = QaServer::MonitorStatus::HistoryPresenter.new(parent: self, historical_summary_data: historical_summary_data)
+      @history_up_down_presenter = QaServer::MonitorStatus::HistoryUpDownPresenter.new(parent: self, historical_up_down_data: historical_up_down_data)
       @performance_presenter = QaServer::MonitorStatus::PerformancePresenter.new(parent: self, performance_data: performance_data)
     end
 
@@ -23,6 +25,9 @@ module QaServer
                    :percent_authority_failing, :percent_authority_failing_str, :failure_style_class, :passing_style_class,
                    :display_history_details?, :display_historical_graph?, :display_historical_datatable?, :history_start, :history_end
 
+    def_delegators :@history_up_down_presenter, :historical_up_down_data, :display_historical_up_down?, :historical_up_down_status_class,
+                   :up_down_start, :up_down_end
+
     def_delegators :@performance_presenter, :performance_data, :performance_data?, :display_performance?, :display_performance_graph?,
                    :display_performance_datatable?, :performance_data_authority_name, :performance_for_day_graph, :performance_for_month_graph,
                    :performance_for_year_graph, :datatable_search_stats, :datatable_fetch_stats, :datatable_all_actions_stats,

data/app/services/qa_server/history_up_down_service.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+# This class determines the state (e.g. fully_up, mostly_up, barely_up, down)of an authority during the last 30 days.
+module QaServer
+  class HistoryUpDownService
+    NO_DATA             = :no_data
+    FULLY_UP            = :fully_up
+    MOSTLY_UP           = :mostly_up
+    EXCESSIVE_TIMEOUTS  = :timeouts
+    BARELY_UP           = :barely_up
+    DOWN                = :down
+
+    MOSTLY_UP_THRESHOLD = QaServer.config.up_down_data_mostly_up_threshold
+    TIMEOUT_THRESHOLD   = QaServer.config.up_down_data_timeouts_max_threshold
+
+    class_attribute :authority_lister, :scenario_history_class, :time_service
+    self.authority_lister = QaServer::AuthorityListerService
+    self.scenario_history_class = QaServer::ScenarioRunHistory
+    self.time_service = QaServer::TimeService
+
+    def last_30_days
+      data = {}
+      authorities_list.each { |authority| data[authority] = last_30_days_for(authority.to_s) }
+      data
+    end
+
+  private
+
+    # @returns [Hash <Array<Hash>>] data for an authority for each of the last 30 days
+    # @example
+    #   { 'AGROVOC' = [
+    #       :FULLY_UP,   # 0 - today
+    #       :MOSTLY_UP,  # 1 - yesterday
+    #       :MOSTLY_UP,  # 2 - two days ago
+    #       :FULLY_UP,   # 3 - three days ago
+    #       :DOWN,       # 4 - four days ago
+    #       ...          # etc.
+    #     ]
+    #   }
+    def last_30_days_for(authority)
+      auth_data = []
+      0.upto(29) { |offset| auth_data[offset] = day_status(authority, offset) }
+      auth_data
+    end
+
+    # @returns [Symbol] status for a given day for an authority
+    def day_status(authority, offset)
+      day = offset_day(offset)
+      good_count = count_good(authority, day)
+      unknown_count = count_unknown(authority, day)
+      bad_count = count_bad(authority, day)
+      timeout_count = count_timeouts(authority, day)
+      status_determination(good_count, unknown_count, bad_count, timeout_count)
+    end
+
+    def status_determination(good_count, unknown_count, bad_count, timeout_count) # rubocop:disable Metrics/CyclomaticComplexity
+      total_count = good_count + unknown_count + bad_count
+      return NO_DATA if total_count.zero?
+      return FULLY_UP if good_count == total_count
+      return DOWN if bad_count == total_count
+      return BARELY_UP if unknown_count == total_count
+      return EXCESSIVE_TIMEOUTS if (timeout_count.to_f / total_count) > TIMEOUT_THRESHOLD
+      return MOSTLY_UP if (bad_count.to_f / total_count) < (1 - MOSTLY_UP_THRESHOLD)
+      BARELY_UP
+    end
+
+    def authorities_list
+      @authorities_list ||= authority_lister.authorities_list
+    end
+
+    def offset_day(offset)
+      @today ||= time_service.current_time
+      time_service.pretty_query_date(@today - offset.days)
+    end
+
+    def count_good(authority, day)
+      scenario_history_class.where(authority_name: authority)
+                            .where(date: day)
+                            .where(status: :good)
+                            .count(:id)
+    end
+
+    def count_unknown(authority, day)
+      scenario_history_class.where(authority_name: authority)
+                            .where(date: day)
+                            .where(status: :unknown)
+                            .count(:id)
+    end
+
+    def count_bad(authority, day)
+      scenario_history_class.where(authority_name: authority)
+                            .where(date: day)
+                            .where(status: :bad)
+                            .count(:id)
+    end
+
+    def count_timeouts(authority, day)
+      scenario_history_class.where(authority_name: authority)
+                            .where(date: day)
+                            .where('err_message LIKE ?', "%timeout%")
+                            .count(:id)
+    end
+  end
+end

data/app/services/qa_server/time_service.rb

@@ -24,6 +24,12 @@ module QaServer
       def pretty_date(dt)
         dt.in_time_zone(QaServer.config.preferred_time_zone_name).strftime("%m/%d/%Y")
       end
+
+      # @param dt [ActiveSupport::TimeWithZone] date time stamp
+      # @return [String] string version of date formatted with just date (e.g. "2020-02-01")
+      def pretty_query_date(dt)
+        dt.in_time_zone(QaServer.config.preferred_time_zone_name).strftime("%Y-%m-%d")
+      end
     end
   end
 end

data/app/views/qa_server/check_status/index.html.erb

@@ -44,8 +44,6 @@
       <select name="authority" id="authority" class="string optional form-control form-control" value="" aria-labelledby="authority" onchange="hide_data()">
         <option value=""><%= t('qa_server.check_status.select_authority') %></option>
         <option disabled>──────────</option>
-        <option value="<%= @presenter.value_all_collections %>"><%= t('qa_server.check_status.show_all') %></option>
-        <option disabled>──────────</option>
         <% @authorities_list.each do |auth_name| %>
           <option value="<%= auth_name %>"<%= " selected" if auth_name == selected_authority %>><%= auth_name.upcase %></option>
         <% end %>
@@ -77,7 +75,10 @@
   <% end %>
 
 
-<div id="status-loading-message" class="wait-message"><%= t('qa_server.check_status.wait_message') %></div>
+<div id="status-loading-message" class="wait-message">
+  <%= t('qa_server.check_status.wait_message_ln1') %><br>
+  <%= t('qa_server.check_status.wait_message_ln2') %>
+</div>
 
 <% if @presenter.connection_status_data? %>
 <div id="connection-status-section" class="status-section">
@@ -100,7 +101,7 @@
           </tr>
       <% end %>
       <tr>
-        <td class="<%= @presenter.status_style_class(status[:status]) %>"><%= @presenter.status_label(status[:status]) %></td>
+        <td class="<%= @presenter.status_style_class(status) %>"><%= @presenter.status_label(status[:status]) %></td>
         <td><%= status[:subauthority_name] %></td>
         <td><%= status[:service] %></td>
         <td><%= status[:action] %></td>
@@ -137,8 +138,8 @@
         </tr>
       <% end %>
     <tr>
-      <td class="position <%= @presenter.status_style_class(status[:status]) %>"><%= status[:expected] %></td>
-      <td class="position <%= @presenter.status_style_class(status[:status]) %>"><%= status[:actual] %></td>
+      <td class="position <%= @presenter.status_style_class(status) %>"><%= status[:expected] %></td>
+      <td class="position <%= @presenter.status_style_class(status) %>"><%= status[:actual] %></td>
       <td><%= status[:request_data] %></td>
       <td><a href="<%= status[:target] %>"><%= status[:target] %></a></td>
       <td><%= status[:authority_name] %></td>

data/app/views/qa_server/monitor_status/_test_summary.html.erb

@@ -17,7 +17,7 @@
       <td class="status-neutral"><%= @presenter.tests_count %></td>
     </tr>
   </table>
-  <p class="status-update-dtstamp"><%= t('qa_server.monitor_status.summary.last_updated', date: @presenter.last_updated) %></p>
+  <p class="status-update-dtstamp"><%= @presenter.last_updated %></p>
 
   <% if @presenter.failures?%>
     <div id="failures" class="status-section">

data/app/views/qa_server/monitor_status/_test_up_down_connection_history.html.erb

@@ -0,0 +1,30 @@
+  <% if @presenter.history? && @presenter.display_history_details?%>
+      <div id="availability-history" class="status-section">
+        <h3><%= t('qa_server.monitor_status.history.title') %></h3>
+        <% if @presenter.display_historical_graph? %>
+          <p class="status-update-dtstamp"><%= t('qa_server.monitor_status.history.range', from: @presenter.history_start, to: @presenter.history_end) %></p>
+          <%= image_tag(@presenter.historical_graph, alt: 'History Graph Unavailable') %>
+        <% end %>
+
+        <% if @presenter.display_historical_up_down? %>
+          <p class="status-update-dtstamp"><%= t('qa_server.monitor_status.history.range', from: @presenter.up_down_start, to: @presenter.up_down_end) %></p>
+          <table class="up-down-history">
+            <tr>
+              <th class="up-down-history"><%= t('qa_server.monitor_status.history.authority') %></th>
+              <% 0.upto(29) do %>
+              <th class='up-down-history'></th>
+              <% end %>
+              <td class='up-down-history'>Most Recent</td>
+            </tr>
+            <% @presenter.historical_up_down_data.each do |authority_name, status| %>
+                <tr>
+                  <td class="connection-up-down"><%= authority_name %></td>
+                  <% 29.downto(0) do |day| %>
+                    <td class="connection-up-down <%= @presenter.historical_up_down_status_class(status, day) %>"></td>
+                  <% end %>
+                </tr>
+            <% end %>
+          </table>
+        <% end %>
+      </div>
+  <% end %>

data/app/views/qa_server/monitor_status/index.html.erb

@@ -5,7 +5,8 @@
   <h2><%= t('qa_server.monitor_status.title') %></h2>
 
   <%= render 'test_summary' %>
-  <%= render 'test_history' %>
+  <%= render 'test_up_down_connection_history' %>
+  <% # = render 'test_history' %>
   <%= render 'performance' %>
 
 </div>

data/config/locales/qa_server.en.yml

@@ -25,12 +25,12 @@ en:
     check_status:
       title:             Check Status
       select_authority:  Select authority...
-      show_all:          ALL Authorities (SLOW)
       connections:       Check Connection Status only
       accuracy:          Check Accuracy only
       comparison:        Compare Accuracy
       all_checks:        Run all checks
-      wait_message:      "Please wait while the status is verified.  This will be slow if you selected ALL Authorities."
+      wait_message_ln1:  "Please wait while the status is verified."
+      wait_message_ln2:  "This may be slow for large authorities or ones with a lot of tests."
       connection_checks: Connection Checks
       accuracy_checks:   Accuracy Checks for Search Results
       comparison_checks: Comparison of Accuracy Checks
@@ -58,7 +58,11 @@ en:
       wait_message:      Fetching term...
       term_results:      Results
     monitor_status:
-      title:             Monitor Status
+      title:               Monitor Status
+      permission_denied:   'Permission denied. Unable to %{action}.'
+      refreshing_tests:    Refreshing tests.
+      refreshing_history:  Refreshing connection history.
+      refreshing_performance: Refreshing performance metrics.
       summary:
         title:           Latest Monitored Status
         summary_table:   Summary
@@ -68,10 +72,10 @@ en:
         failing_tests:   Failing Tests
         total_tests:     TOTAL Tests
         last_updated:    'Status last updated %{date} ET'
+        no_data:         'No Data Available (Common Cause: first time tests are running)'
       failures:
         title:           Failures During Latest Status Update
         status:          Status
-        authority:       Authority
         subauthority:    Subauthority
         service:         Service
         action:          Action
@@ -79,7 +83,6 @@ en:
         errmsg:          Errors
       history:
         title:           Authority Connection History
-        since:           'Since %{date} ET'
         range:           'From %{from} to %{to}'
         authority:       Authority
         days_failing:    Days Failing
@@ -100,7 +103,6 @@ en:
         graph_load_time_ms:    Graph Load Time (ms)
         normalization_time_ms: Normalization Time (ms)
         retrieve_time_ms:      Retrieve Time (ms)
-        full_request_time_ms:  Full Request Time (ms)
         x_axis_hour:     Hour
         x_axis_day:      Day
         x_axis_month:    Month
@@ -113,7 +115,6 @@ en:
         datatable_month_desc: 'Month'
         datatable_year_desc:  'Year'
         datarange:            'From %{from} to %{to}'
-        authority:            Authority
         average_times:        Average (ms)
         fastest_times:        10th percentile (ms)
         slowest_times:        90th percentile (ms)

data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json

@@ -3,8 +3,8 @@
   "service_uri": "http://ld4l.org/ld4l_services/cache",
   "prefixes": {
     "getty":   "http://vocab.getty.edu/ontology#",
-    "vivo":    "http://vivoweb.org/ontology/core#",
-    "xl":      "http://www.w3.org/2008/05/skos-xl#"
+    "skosxl":  "http://www.w3.org/2008/05/skos-xl#",
+    "vivo":    "http://vivoweb.org/ontology/core#"
   },
   "term": {
     "url": {
@@ -28,11 +28,9 @@
     "term_id": "URI",
     "results": {
       "id_ldpath":       "dc:identifier ::xsd:string",
-      "label_ldpath":    "skos:prefLabel ::xsd:string",
-      "altlabel_ldpath": "skos:altLabel ::xsd:string",
-      "broader_ldpath":  "skos:broader ::xsd:anyURI",
-      "narrower_ldpath": "skos:narrower ::xsd:anyURI",
-      "sameas_ldpath":   "skos:exactMatch ::xsd:anyURI"
+      "label_ldpath":    "^foaf:focus/skosxl:prefLabel/skosxl:literalForm :: xsd:string",
+      "altlabel_ldpath": "^foaf:focus/skosxl:altLabel/skosxl:literalForm :: xsd:string",
+      "broader_ldpath":  "^foaf:focus/getty:broaderPreferred ::xsd:anyURI"
     }
   },
   "search": {
@@ -79,8 +77,7 @@
     },
     "total_count_ldpath": "vivo:count",
     "results": {
-      "id_ldpath":    "dc:identifier ::xsd:string",
-      "label_ldpath": "skos:prefLabel ::xsd:string",
+      "label_ldpath": "^foaf:focus/skosxl:prefLabel/skosxl:literalForm :: xsd:string",
       "sort_ldpath":  "vivo:rank ::xsd:string"
     },
     "context": {
@@ -88,28 +85,42 @@
         {
           "property_label_i18n": "qa.linked_data.authority.getty_tgn_ld4l_cache.preferred_label",
           "property_label_default": "Preferred label",
-          "ldpath": "^foaf:focus/xl:prefLabel/xl:literalForm :: xsd:string",
+          "ldpath": "^foaf:focus/skosxl:prefLabel/skosxl:literalForm :: xsd:string",
           "selectable": true,
           "drillable": false
         },
         {
           "property_label_i18n": "qa.linked_data.authority.getty_tgn_ld4l_cache.variant_label",
           "property_label_default": "Variant Label",
-          "ldpath": "^foaf:focus/xl:altLabel/xl:literalForm :: xsd:string",
+          "ldpath": "^foaf:focus/skosxl:altLabel/skosxl:literalForm :: xsd:string",
+          "selectable": false,
+          "drillable": false
+        },
+        {
+          "property_label_i18n": "qa.linked_data.authority.getty_tgn_ld4l_cache.parent_body",
+          "property_label_default": "Parent body",
+          "ldpath": "^foaf:focus/getty:parentString :: xsd:string",
+          "selectable": false,
+          "drillable": false
+        },
+        {
+          "property_label_i18n": "qa.linked_data.authority.getty_tgn_ld4l_cache.parent_body_abbreviation",
+          "property_label_default": "Parent body (abbreviation)",
+          "ldpath": "^foaf:focus/getty:parentStringAbbrev :: xsd:string",
           "selectable": false,
           "drillable": false
         },
         {
           "property_label_i18n": "qa.linked_data.authority.getty_tgn_ld4l_cache.place_type",
           "property_label_default": "Place type",
-          "ldpath": "^foaf:focus/gvp:placeTypePreferred :: xsd:string",
+          "ldpath": "^foaf:focus/getty:placeTypePreferred :: xsd:string",
           "selectable": false,
           "drillable": false
         },
         {
           "property_label_i18n": "qa.linked_data.authority.getty_aat_ld4l_cache.broader",
           "property_label_default": "Broader",
-          "ldpath": "^foaf:focus/gvp:broaderPreferred/xl:altLabel/xl:literalForm :: xsd:string",
+          "ldpath": "^foaf:focus/getty:broaderPreferred :: xsd:string",
           "selectable": false,
           "drillable": true,
           "expansion_label_ldpath": "skos:prefLabel ::xsd:string",
@@ -119,4 +130,3 @@
     }
   }
 }
-