warclight 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12103e04edc1b70c407b06bbd03450d15691fc67
-  data.tar.gz: 5e4dd61e8ff23c5541a6656fc37b600f6d847817
+  metadata.gz: a8ae7e47fad95aad7c598fb0eeba1d729bf35641
+  data.tar.gz: 47865f795f928fdeba81e30e557f0b9e9f19184c
 SHA512:
-  metadata.gz: 5d5ab472de5c1f8d8ace1ab3614a82bbfae56796078b534bc5f7040f0360a6df3c46aaa1b9b85eab4c2c6cb87cdfedd185b115b3f5b9c860e82db7ceafa34bcc
-  data.tar.gz: 6bbe3ec8979d2a169b332e1d8271d2bdce075a162704ee2bad240bdd827bf8af901ccb746d72cb46675cef330a43ee8c820d8586a63d63273b59a2f665e3cfcb
+  metadata.gz: 3f8ad4ff95368d0dc27d1235282c23fe88c95d3a2d27d8da1a9849fd0bcfa073232885b5faf10f0a4fe38337c8da05f19b3e3248e74568e83c461e2fe811dac8
+  data.tar.gz: b488e29c86bb559189081edf5ee8d95a6f6a401da8d501cba50ef0df701c1b97b9ff3defbab81b46f3c55b30bdc234dd69a8cc8c057e92368aca0b58523bd579

data/.rubocop.yml

@@ -67,3 +67,7 @@ Style/Documentation:
 
 Bundler/DuplicatedGem:
   Enabled: false
+
+Lint/UselessAssignment:
+  Exclude:
+    - 'app/models/concerns/warclight/solr_document.rb' #Come back to this later

data/.travis.yml

@@ -3,9 +3,12 @@ language: ruby
 notifications:
   email: false
 rvm:
-  - 2.4.1
+  - 2.4.2
+  - 2.3.5
+  - 2.2.8
 jdk:
   - oraclejdk8
+  - openjdk8
 
 # Update to phantomjs 2.1.1 to fix issues with < 2
 # See https://github.com/travis-ci/travis-ci/issues/3225#issuecomment-177592725

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## [v0.5.0](https://github.com/archivesunleashed/warclight/tree/v0.5.0) (2017-10-19)
+[Full Changelog](https://github.com/archivesunleashed/warclight/compare/v0.4.0...v0.5.0)
+
+**Merged pull requests:**
+
+- Align with https://github.com/ukwa/webarchive-discovery/pull/129 [\#29](https://github.com/archivesunleashed/warclight/pull/29) ([ruebot](https://github.com/ruebot))
+
 ## [v0.4.0](https://github.com/archivesunleashed/warclight/tree/v0.4.0) (2017-10-18)
 [Full Changelog](https://github.com/archivesunleashed/warclight/compare/v0.3.0...v0.4.0)
 

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,81 @@
+# Archives Unleashed Project Code of Conduct
+
+## Our Pledge
+
+* The Archives Unleashed Project believes in supporting an open, inclusive, and
+ diverse community which respects the experience, expertise, and knowledge of
+all community members.
+* The Archives Unleashed community is dedicated to providing a harassment-free
+ experience for everyone, and welcomes individuals regardless age, body size,
+disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+* To foster respectful collaborations this code of conduct applies to all
+Archives Unleashed spaces, includes, but is not limited to, GitHub, Slack,
+Medium, social media platforms and meeting spaces, both online and off.
+* Anyone who violates this code of conduct may be sanctioned or expelled from
+ these spaces at the discretion of the Archives Unleashed Project Team.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at archivesunleashed@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/README.md

@@ -10,7 +10,7 @@ A [Project Blacklight](http://projectblacklight.org/) based [Rails engine](http:
 ## Requirements
 
 * [Ruby](https://www.ruby-lang.org/en/) 2.2 or later
-* [Rails](http://rubyonrails.org) 5.1.2 or later
+* [Rails](http://rubyonrails.org) 5.0.0 or later
 
 ## Installation
 

data/app/helpers/warclight_helper.rb

@@ -19,6 +19,10 @@ module WarclightHelper
   end
 
   def return_five(options = {})
-    options[:value][0, 5].join('; ') + '...'
+    if options[:value].length > 5
+      options[:value][0, 5].join('; ') + '...'
+    else
+      options[:value].join('; ')
+    end
   end
 end

data/app/models/concerns/warclight/solr_document.rb

@@ -8,15 +8,17 @@ module Warclight
 
     def replay_link
       time_travel_base_url = 'http://timetravel.mementoweb.org/api/json/'
-      time_travel_time_format = '%Y%m%d%H%M%S'
-      time_travel_time = (Time.parse(first(:crawl_date)).strftime time_travel_time_format).to_s
-      time_travel_request_url = time_travel_base_url + time_travel_time + '/' + first(:url).to_s
+      time_travel_request_url = time_travel_base_url + first(:wayback_date).to_s + '/' + first(:url).to_s
       time_travel_request = URI(time_travel_request_url)
       time_travel_response = Net::HTTP.get(time_travel_request)
-      time_travel_response_json = JSON.parse(time_travel_response)
-      replay_url = time_travel_response_json['mementos']['closest']['uri'][0]
-      replay_url_link = '<a href="' + "#{replay_url}" '" target="_blank">'"#{replay_url}"'</a> 🔗'
-      replay_url_link.html_safe
+      if time_travel_response.present?
+        time_travel_response_json = JSON.parse(time_travel_response)
+        replay_url = time_travel_response_json['mementos']['closest']['uri'][0]
+        replay_url_link = '<a href="' + "#{replay_url}" '" target="_blank">'"#{replay_url}"'</a> 🔗'
+        replay_url_link.html_safe
+      else
+        replay_url = 'Not Available.'
+      end
     end
   end
 end

data/lib/generators/warclight/install_generator.rb

@@ -19,10 +19,6 @@ module Warclight
       end
     end
 
-    def install_blacklight_range_limit
-      generate 'blacklight_range_limit:install'
-    end
-
     def add_custom_routes
       inject_into_file 'config/routes.rb', after: "mount Blacklight::Engine => '/'" do
         "\n    mount Warclight::Engine => '/'\n"
@@ -31,7 +27,6 @@ module Warclight
 
     def assets
       copy_file 'warclight.scss', 'app/assets/stylesheets/warclight.scss'
-      copy_file 'warclight.js', 'app/assets/javascripts/warclight.js'
       inject_into_file 'app/assets/javascripts/application.js', after: '//= require blacklight/blacklight' do
         "\n//= require bootstrap/scrollspy\n" \
         "\n//= require bootstrap/tab\n"

data/lib/generators/warclight/templates/catalog_controller.rb

@@ -48,13 +48,13 @@ class CatalogController < ApplicationController
     # :index_range can be an array or range of prefixes that will be used to
     # create the navigation (note: It is case sensitive when searching values)
 
-    config.add_facet_field 'content_type_norm', label: 'General Content Type', collapse: false, limit: true
-    config.add_facet_field 'crawl_year', label: 'Crawl Year', collapse: false, limit: true, sort: 'index'
+    config.add_facet_field 'content_type_norm', label: 'General Content Type', collapse: false
+    config.add_facet_field 'crawl_year', label: 'Crawl Year', collapse: false, sort: 'index'
     config.add_facet_field 'public_suffix', label: 'Public Suffix', collapse: false, limit: true
     config.add_facet_field 'domain', label: 'Domain', limit: true
     config.add_facet_field 'links_domains', label: 'Links Domains', limit: true
     config.add_facet_field 'content_language', label: 'Content Language', limit: true
-    config.add_facet_field 'resourcename_s', label: 'Resource Name', limit: true
+    config.add_facet_field 'resourcename_facet', label: 'Resource Name', limit: true
     config.add_facet_field 'institution', label: 'Institution'
     config.add_facet_field 'collection', label: 'Collection'
     config.add_facet_field 'collection_id', label: 'Collection Id'

data/lib/warclight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Warclight
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/solr/conf/schema.xml

@@ -24,7 +24,7 @@
         <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
 
         <!--Not defined in SolrFields. Schema only defintion -->
-        <field name="index_time" type="date" indexed="true" stored="false" docValues="true" default="NOW" /> 
+        <field name="index_time" type="date" indexed="true" stored="false" docValues="true" default="NOW" />
 
         <!-- BL UKWA: additional -->
         <field name="access_terms" type="string" indexed="true" docValues="true" multiValued="true"/>
@@ -76,7 +76,7 @@
         <field name="last_modified" type="tdate" indexed="true" stored="true" docValues="true"/>
         <field name="last_modified_year" type="string" indexed="true" docValues="true"/>
         <field name="license_url" type="string" indexed="true" docValues="true" multiValued="true"/>
-        <field name="links_images" type="string" indexed="true" docValues="true" multiValued="true"/>
+        <field name="links_images" type="text_general" indexed="true" stored="true" multiValued="true"/>
         <field name="links_domains" type="string" indexed="true" docValues="true" multiValued="true"/>
         <field name="links_hosts" type="string" indexed="true" docValues="true" multiValued="true"/>
         <field name="links_hosts_surts" type="string" indexed="true" docValues="true" multiValued="true"/>
@@ -85,7 +85,7 @@
         <field name="links" type="string" indexed="true" docValues="true" multiValued="true"/>
         <field name="locations" type="location" indexed="true" stored="true" multiValued="true"/>
         <field name="parse_error" type="string" indexed="true" docValues="true" multiValued="true"/>
-        <field name="pdf_pdfa_errors" type="string" indexed="true" docValues="true" multiValued="true"/>
+        <field name="pdf_pdfa_errors" type="text_general" indexed="false" stored="true" multiValued="true"/>
         <field name="pdf_pdfa_is_valid" type="string" indexed="true" docValues="true" multiValued="false"/>
         <field name="postcode_district" type="string" indexed="true" docValues="true" multiValued="true"/>
         <field name="postcode" type="string" indexed="true" docValues="true" multiValued="true"/>
@@ -93,8 +93,10 @@
         <field name="publication_year" type="string" indexed="true" docValues="true" multiValued="false"/>
         <field name="public_suffix" type="string" indexed="true" docValues="true" multiValued="false"/>
         <field name="record_type" type="string" indexed="true" stored="true" multiValued="false" docValues="true"/>
+        <field name="redirect_to_norm" type="string" indexed="true" stored="false" docValues="true" multiValued="false"/>
         <field name="referrer_url" type="string" indexed="true" stored="true" multiValued="false" docValues="true"/>
         <field name="resourcename" type="text_general" indexed="true" stored="true"/>
+        <field name="resourcename_facet" type="string" indexed="false" stored="false" docValues="true" multiValued="false"/>
         <field name="sentiment_score" type="float" indexed="true" stored="true" multiValued="false"/>
         <field name="sentiment" type="string" indexed="true" docValues="true" multiValued="false"/>
         <field name="server" type="string" indexed="true" docValues="true" multiValued="true"/>
@@ -192,7 +194,7 @@
         <copyField source="wct_description" dest="text"/>
         <copyField source="url" dest="text"/>
         <copyField source="content" dest="text"/>
-        <copyField source="resourcename" dest="resourcename_s" indexed="false" stored="false" docValues="true"/>
+        <copyField source="resourcename" dest="resourcename_facet"/>
 
   <types>
         <fieldType name="string" class="solr.StrField" sortMissingLast="true" />

data/solr/warclight_warc-indexer.conf

@@ -0,0 +1,170 @@
+{
+	"warc": {
+		"title": "Default Warclight warc-indexer config.",
+		"http_proxy": {},
+		"solr": {
+			"batch_size": 500,
+			"num_shards": 1,
+			"dummy_run": false,
+			"num_threads": 1,
+			"disablecommit": true,
+			"use_hash_url_id": true,
+			"check_solr_for_duplicates": true,
+
+                        "field_setup": {
+                            "fields" : {
+                                "url" : { "max_length" : 2048 },
+                                "url_norm" : { "max_length" : 2048 },
+                                "links" : { "max_length" : 2048 },
+                                "content" : { "max_length" : 512K },
+                            },
+                        },
+		},
+		"index": {
+			"tika": {
+				"parse_timeout": 300000,
+				"use_boilerpipe": false,
+				"extract_all_metadata": false,
+				"exclude_mime": [
+					"x-tar",
+					"x-gzip",
+					"bz",
+					"lz",
+					"compress",
+					"zip",
+					"javascript",
+					"css",
+					"octet-stream"
+				],
+				"max_text_length": "512K"
+			},
+			"id": {
+				"useResourceURI": true,
+				"droid": {
+					"enabled": true,
+					"useBinarySignaturesOnly": false
+				}
+			},
+			"exclusions": {
+				"enabled": false,
+				"check_interval": 600,
+				"file": "/path/to/exclude.txt"
+			},
+			"extract": {
+				"onDiskThreshold": "500M",
+				"response_include": [
+					"1",
+					"2",
+					"3",
+					"4",
+					"5",
+					"6",
+					"7",
+					"8",
+					"9"
+				],
+				"url_exclude": [],
+				"protocol_include": [
+					"http",
+					"https"
+				],
+				"record_type_include": [
+					"response",
+					"revisit"
+				],
+				"inMemoryThreshold": "500M",
+				"content": {
+					"text": true,
+					"text_stored": true,
+					"elements_used": true,
+					"first_bytes": {
+						"num_bytes": 32,
+						"enabled": true
+					},
+					"annotations": {
+						"enabled": false,
+						"file": "/path/to/annotations.json",
+						"surt_prefix_file": "/path/to/openAccessSurts.txt"
+					},
+					"images": {
+						"dominantColours": false,
+						"maxSizeInBytes": "50M",
+						"detectFaces": false,
+						"analysisSamplingRate": 1,
+						"enabled": true
+					},
+					"extractApachePreflightErrors": false,
+					"text_fuzzy_hash": true,
+					"text_extract_postcodes": false,
+					"language": {
+						"langdetectprofiles": [
+							"en",
+							"fr",
+							"af",
+							"ar",
+							"bg",
+							"bn",
+							"cs",
+							"de",
+							"el",
+							"es",
+							"et",
+							"fa",
+							"fi",
+							"da",
+							"gu",
+							"he",
+							"hi",
+							"hr",
+							"hu",
+							"id",
+							"it",
+							"ja",
+							"kn",
+							"ko",
+							"lt",
+							"lv",
+							"mk",
+							"ml",
+							"mr",
+							"ne",
+							"nl",
+							"no",
+							"pa",
+							"pl",
+							"pt",
+							"ro",
+							"ru",
+							"sk",
+							"sl",
+							"so",
+							"sq",
+							"sv",
+							"sw",
+							"ta",
+							"te",
+							"th",
+							"tl",
+							"tr",
+							"uk",
+							"ur",
+							"vi",
+							"zh-cn",
+							"zh-tw"
+						],
+						"enabled": true
+					},
+					"test_sentimentj": false,
+					"text_stanford_ner": false
+				},
+				"linked": {
+					"hosts": true,
+					"resources": true,
+					"domains": true,
+					"normalise": true,
+					"images": true
+				}
+			}
+		}
+	}
+}

data/tasks/warclight.rake

@@ -56,6 +56,7 @@ namespace :warclight do
     puts 'Seeding index with data from spec/fixtures/warcs/...'
     # rubocop:disable Metrics/LineLength
     system('curl -o ".internal_test_app/tmp/warc-indexer.jar" "http://alpha.library.yorku.ca/warc-indexer-3.0.0-SNAPSHOT-jar-with-dependencies.jar"')
-    system('java -Djava.io.tmpdir=.internal_test_app/tmp -jar .internal_test_app/tmp/warc-indexer.jar -i "York University Libraries" -n "Test Collection" -u "12345" -s http://localhost:8983/solr/blacklight-core spec/fixtures/warcs/*.gz')
+    system('java -Djava.io.tmpdir=.internal_test_app/tmp -jar .internal_test_app/tmp/warc-indexer.jar -c .internal_test_app/solr/warclight_warc-indexer.conf -i "York University Libraries" -n "Test Collection" -u "12345" -s http://localhost:8983/solr/blacklight-core spec/fixtures/warcs/*.gz')
+    system('curl "http://localhost:8983/solr/blacklight-core/update?commit=true&openSearcher=true"')
   end
 end

data/template.rb

@@ -3,7 +3,6 @@
 # We can remove `blacklight` and `blacklight_range_limit` once they are released
 # since they are already dependents of Warclight (but we need to pin to particular branches)
 gem 'blacklight', github: 'projectblacklight/blacklight'
-gem 'blacklight_range_limit', github: 'projectblacklight/blacklight_range_limit', branch: 'blacklight-7'
 
 gem 'warclight', github: 'archivesunleashed/warclight'
 

data/warclight.gemspec

@@ -22,11 +22,10 @@ Gem::Specification.new do |s|
   s.require_paths   = ['lib']
 
   s.add_dependency 'blacklight'
-  s.add_dependency 'blacklight_range_limit'
-  s.add_dependency 'rails', '~> 5.1.2'
+  s.add_dependency 'rails', '~> 5.0'
 
   s.add_development_dependency 'bundler', '~> 1.14'
-  s.add_development_dependency 'engine_cart'
+  s.add_development_dependency 'engine_cart', '~> 1.2.0'
   s.add_development_dependency 'mocha'
   s.add_development_dependency 'poltergeist'
   s.add_development_dependency 'rake', '~> 12.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: warclight
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Nick Ruest
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-19 00:00:00.000000000 Z
+date: 2018-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: blacklight
@@ -24,34 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: blacklight_range_limit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.2
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.2
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +56,16 @@ dependencies:
   name: engine_cart
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +192,7 @@ files:
 - ".solr_wrapper"
 - ".travis.yml"
 - CHANGELOG.md
+- CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE.txt
@@ -213,7 +200,6 @@ files:
 - Rakefile
 - app/assets/images/blacklight/compact.svg
 - app/assets/images/blacklight/logo.png
-- app/assets/javascripts/warclight/warclight.js
 - app/assets/stylesheets/warclight/application.scss
 - app/assets/stylesheets/warclight/warclight.scss
 - app/controllers/concerns/warclight/field_config_helpers.rb
@@ -222,7 +208,6 @@ files:
 - app/models/concerns/warclight/catalog.rb
 - app/models/concerns/warclight/search_behavior.rb
 - app/models/concerns/warclight/solr_document.rb
-- app/views/layouts/warclight/application.html.erb
 - bin/rails
 - config/routes.rb
 - lib/generators/warclight/install_generator.rb
@@ -279,6 +264,7 @@ files:
 - solr/conf/solrconfig.xml
 - solr/conf/stopwords.txt
 - solr/conf/synonyms.txt
+- solr/warclight_warc-indexer.conf
 - tasks/warclight.rake
 - template.rb
 - vendor/assets/javascripts/responsiveTruncator.js
@@ -304,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: A Rails engine supporting discovery of web archives.

data/app/assets/javascripts/warclight/warclight.js

@@ -1,2 +0,0 @@
-// Vendor Scripts
-//= require responsiveTruncator

data/app/views/layouts/warclight/application.html.erb

@@ -1,14 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>Warclight</title>
-  <%= stylesheet_link_tag    "warclight/application", media: "all" %>
-  <%= javascript_include_tag "warclight/application" %>
-  <%= csrf_meta_tags %>
-</head>
-<body>
-
-<%= yield %>
-
-</body>
-</html>