arclight 0.4.0 → 1.0.0.beta1

data/lib/tasks/index.rake

@@ -18,26 +18,23 @@ require 'benchmark'
 #
 namespace :arclight do
   desc 'Index an EAD document, use FILE=<path/to/ead.xml> and REPOSITORY_ID=<myid>'
-  task :index do
+  # We need :environment to have access to the Blacklight confg
+  task index: :environment do
     raise 'Please specify your EAD document, ex. FILE=<path/to/ead.xml>' unless ENV['FILE']
 
-    print "Loading #{ENV['FILE']} into index...\n"
-    solr_url = begin
-                 Blacklight.default_index.connection.base_uri
-               rescue StandardError
-                 ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
-               end
-    elapsed_time = Benchmark.realtime {
-      `bundle exec traject -u #{solr_url} -i xml -c #{Arclight::Engine.root}/lib/arclight/traject/ead2_config.rb #{ENV['FILE']}`
-    }
-    print "Indexed #{ENV['FILE']} (in #{elapsed_time.round(3)} secs).\n"
+    print "Loading #{ENV.fetch('FILE', nil)} into index...\n"
+    solr_url = ENV.fetch('SOLR_URL', Blacklight.default_index.connection.base_uri)
+    elapsed_time = Benchmark.realtime do
+      `bundle exec traject -u #{solr_url} -i xml -c #{Arclight::Engine.root}/lib/arclight/traject/ead2_config.rb #{ENV.fetch('FILE', nil)}`
+    end
+    print "Indexed #{ENV.fetch('FILE', nil)} (in #{elapsed_time.round(3)} secs).\n"
   end
 
   desc 'Index a directory of EADs, use DIR=<path/to/directory> and REPOSITORY_ID=<myid>'
   task :index_dir do
     raise 'Please specify your directory, ex. DIR=<path/to/directory>' unless ENV['DIR']
 
-    Dir.glob(File.join(ENV['DIR'], '*.xml')).each do |file|
+    Dir.glob(File.join(ENV.fetch('DIR', nil), '*.xml')).each do |file|
       system("rake arclight:index FILE=#{file}")
     end
   end
@@ -46,7 +43,7 @@ namespace :arclight do
   task :index_url do
     raise 'Please specify your EAD document, ex. URL=<http[s]://domain/path/to/ead.xml>' unless ENV['URL']
 
-    ead = Nokogiri::XML(open(ENV['URL']))
+    ead = Nokogiri::XML(open(ENV.fetch('URL', nil)))
     tmp = Tempfile.new(["#{Time.now.to_i}-", '.xml'], encoding: 'utf-8')
     begin
       tmp.write ead
@@ -64,15 +61,15 @@ namespace :arclight do
   task :index_url_batch do
     raise 'Please specify your URLs file, ex. BATCH=<path/to/urls.txt>' unless ENV['BATCH']
 
-    File.open(ENV['BATCH']).each_line do |l|
+    File.open(ENV.fetch('BATCH', nil)).each_line do |l|
       ENV['URL'] = l.chomp
       next if ENV['URL'].empty?
 
-      unless ENV['URL'] =~ /\A#{URI.regexp(%w[http https])}\z/
-        puts "Skipping invalid looking url #{ENV['URL']}"
+      unless ENV.fetch('URL', nil) =~ /\A#{URI::DEFAULT_PARSER.make_regexp(%w[http https])}\z/
+        puts "Skipping invalid looking url #{ENV.fetch('URL', nil)}"
         next
       end
-      puts "Indexing #{ENV['URL']}"
+      puts "Indexing #{ENV.fetch('URL', nil)}"
       Rake::Task['arclight:index_url'].invoke
       Rake::Task['arclight:index_url'].reenable
     end

data/package.json

@@ -1,16 +1,14 @@
 {
   "name": "arclight",
-  "description": "",
-  "main": "index.js",
-  "dependencies": {
-    "@babel/core": "^7.5.5",
-    "@babel/plugin-external-helpers": "^7.2.0",
-    "@babel/plugin-transform-modules-umd": "^7.2.0",
-    "@babel/preset-env": "^7.5.5"
-  },
+  "version": "1.0.0-beta1",
+  "description": "The frontend for arclight",
+  "main": "app/assets/javascript/arclight/arclight.js",
+  "files": [
+    "app/assets"
+  ],
   "devDependencies": {
-    "eslint": "^3.19.0",
-    "eslint-config-airbnb-base": "^11.1.3",
+    "eslint": "^8.26.0",
+    "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.2.0"
   },
   "scripts": {
@@ -20,12 +18,12 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/sul-dlss/arclight.git"
+    "url": "git+https://github.com/projectblacklight/arclight.git"
   },
   "author": "",
   "license": "Apache-2.0",
   "bugs": {
-    "url": "https://github.com/sul-dlss/arclight/issues"
+    "url": "https://github.com/projectblacklight/arclight/issues"
   },
-  "homepage": "https://github.com/sul-dlss/arclight#readme"
+  "homepage": "https://github.com/projectblacklight/arclight#readme"
 }

data/solr/conf/schema.xml

@@ -150,7 +150,8 @@
       <analyzer type="index">
         <tokenizer class="solr.WhitespaceTokenizerFactory" />
         <filter class="solr.KeywordRepeatFilterFactory" />
-        <filter class="solr.WordDelimiterGraphFilterFactory"/>
+        <filter class="solr.WordDelimiterGraphFilterFactory" />
+        <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" />
         <filter class="solr.EnglishPossessiveFilterFactory"/>
         <filter class="solr.ICUFoldingFilterFactory" />
         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
@@ -162,7 +163,7 @@
         <tokenizer class="solr.WhitespaceTokenizerFactory" />
         <filter class="solr.KeywordRepeatFilterFactory" />
         <filter class="solr.WordDelimiterGraphFilterFactory"/>
-        <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+        <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" />
         <filter class="solr.ICUFoldingFilterFactory" />
         <filter class="solr.EnglishPossessiveFilterFactory"/>
         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
@@ -190,7 +191,6 @@
           catenateAll="1"
           />
         <filter class="solr.ICUFoldingFilterFactory" />
-        <filter class="solr.WordDelimiterGraphFilterFactory"/>
       </analyzer>
     </fieldType>
 
@@ -244,8 +244,8 @@
       </analyzer>
     </fieldType>
 
-    <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
-    <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+    <!-- A specialized field for geospatial search filters and distance sorting. -->
+    <fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true" />
 
     <fieldType name="_nest_path_" class="solr.NestPathField" />
 
@@ -280,9 +280,9 @@
    <field name="id" type="string" indexed="true" stored="true" required="true" />
    <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
    <field name="timestamp" type="pdate" indexed="true" stored="true" default="NOW" multiValued="false"/>
-   <!-- default, catch all search field -->
+   <!-- default, catch all search field, also used for search term highlighting (requires stored="true") -->
    <field name="text" type="text" indexed="true" stored="true" multiValued="true"/>
-   <field name="unitid_identifier_match" type="identifier_match" indexed="true" multiValued="true" />
+   <field name="unitid_identifier_match" type="identifier_match" indexed="true" stored="false" multiValued="true" />
 
    <field name="_root_" type="string" indexed="true" stored="true" docValues="false" />
    <field name="_nest_parent_" type="string" indexed="true" stored="true"/>
@@ -307,7 +307,9 @@
         unknown fields indexed and/or stored by default -->
    <!--dynamicField name="*" type="ignored" multiValued="true" /-->
 
+   <!-- To play nice with atomic updates, only use non-stored fields for copyFields -->
    <dynamicField name="*_tesim" type="text_en"   stored="true"  indexed="true"  multiValued="true"  />
+   <dynamicField name="*_tesm" type="text_en"    stored="true"  indexed="false" multiValued="true"  />
    <dynamicField name="*_teim"  type="text_en"   stored="false" indexed="true"  multiValued="true"  />
    <dynamicField name="*_si"    type="string"    stored="false" indexed="true"  multiValued="false" />
    <dynamicField name="*_sim"   type="string"    stored="false" indexed="true"  multiValued="true"  />
@@ -315,7 +317,9 @@
    <dynamicField name="*_ssi"   type="string"    stored="true"  indexed="true"  multiValued="false" />
    <dynamicField name="*_ssim"  type="string"    stored="true"  indexed="true"  multiValued="true"  />
    <dynamicField name="*_isim"  type="pint"      stored="true"  indexed="true"  multiValued="true"  />
+   <dynamicField name="*_is"    type="pint"      stored="true"  indexed="false" multiValued="false" />
    <dynamicField name="*_ii"    type="pint"      stored="false" indexed="true"  multiValued="false" />
+   <dynamicField name="*_isi"   type="pint"      stored="true"  indexed="true"  multiValued="false" />
  </fields>
 
  <!-- Field to use to determine and enforce document uniqueness.
@@ -330,42 +334,42 @@
    <!-- Copy Fields -->
 
    <!-- field-based searches -->
-   <copyField source="normalized_title_ssm" dest="title_tesim"/>
-   <copyField source="places_ssm" dest="place_tesim"/>
-   <copyField source="names_ssim" dest="name_tesim"/>
-   <copyField source="access_subjects_ssim" dest="subject_tesim"/>
+   <copyField source="normalized_title_ssm" dest="normalized_title_teim"/>
+   <copyField source="places_ssim" dest="place_teim"/>
+   <copyField source="names_ssim" dest="name_teim"/>
+   <copyField source="access_subjects_ssim" dest="subject_teim"/>
 
    <!-- The catch all `text` field -->
    <!-- grab the fielded searches -->
    <copyField source="normalized_title_ssm" dest="text" />
-   <copyField source="places_ssm" dest="text" />
+   <copyField source="places_ssim" dest="text" />
    <copyField source="names_ssim" dest="text" />
    <copyField source="access_subjects_ssim" dest="text" />
    <!-- grab the searchable notes -->
-   <copyField source="abstract_teim" dest="text" />
-   <copyField source="accessrestricct_teim" dest="text" />
-   <copyField source="accruals_teim" dest="text" />
-   <copyField source="acqinfo_teim" dest="text" />
-   <copyField source="altformavail_teim" dest="text" />
-   <copyField source="appraisal_teim" dest="text" />
-   <copyField source="arrangement_teim" dest="text" />
-   <copyField source="bibliography_teim" dest="text" />
-   <copyField source="bioghist_teim" dest="text" />
-   <copyField source="custodhist_teim" dest="text" />
-   <copyField source="did_note_teim" dest="text" />
-   <copyField source="fileplan_teim" dest="text" />
-   <copyField source="materialspec_teim" dest="text" />
-   <copyField source="note_teim" dest="text" />
-   <copyField source="odd_teim" dest="text" />
-   <copyField source="originalsloc_teim" dest="text" />
-   <copyField source="physdesc_teim" dest="text" />
-   <copyField source="physloc_teim" dest="text" />
-   <copyField source="phystech_teim" dest="text" />
-   <copyField source="processinfo_teim" dest="text" />
-   <copyField source="relatedmaterial_teim" dest="text" />
-   <copyField source="scopecontent_teim" dest="text" />
-   <copyField source="separatedmaterial_teim" dest="text" />
-   <copyField source="userestrict_teim" dest="text" />
+   <copyField source="abstract_tesim" dest="text" />
+   <copyField source="accessrestricct_tesim" dest="text" />
+   <copyField source="accruals_tesim" dest="text" />
+   <copyField source="acqinfo_tesim" dest="text" />
+   <copyField source="altformavail_tesim" dest="text" />
+   <copyField source="appraisal_tesim" dest="text" />
+   <copyField source="arrangement_tesim" dest="text" />
+   <copyField source="bibliography_tesim" dest="text" />
+   <copyField source="bioghist_tesim" dest="text" />
+   <copyField source="custodhist_tesim" dest="text" />
+   <copyField source="did_note_tesim" dest="text" />
+   <copyField source="fileplan_tesim" dest="text" />
+   <copyField source="materialspec_tesim" dest="text" />
+   <copyField source="note_tesim" dest="text" />
+   <copyField source="odd_tesim" dest="text" />
+   <copyField source="originalsloc_tesim" dest="text" />
+   <copyField source="physdesc_tesim" dest="text" />
+   <copyField source="physloc_tesim" dest="text" />
+   <copyField source="phystech_tesim" dest="text" />
+   <copyField source="processinfo_tesim" dest="text" />
+   <copyField source="relatedmaterial_tesim" dest="text" />
+   <copyField source="scopecontent_tesim" dest="text" />
+   <copyField source="separatedmaterial_tesim" dest="text" />
+   <copyField source="userestrict_tesim" dest="text" />
    <!-- grab structured data that's important -->
    <copyField source="unitid_ssm" dest="text" />
    <copyField source="unitid_ssm" dest="unitid_identifier_match" />
@@ -381,9 +385,9 @@
 
    <!-- for suggestions -->
    <copyField source="normalized_title_ssm" dest="suggest"/>
-   <copyField source="collection_sim" dest="suggest"/>
+   <copyField source="collection_ssim" dest="suggest"/>
    <copyField source="names_ssim" dest="suggest"/>
-   <copyField source="repository_sim" dest="suggest"/>
+   <copyField source="repository_ssim" dest="suggest"/>
    <copyField source="places_ssim" dest="suggest"/>
    <copyField source="access_subjects_ssim" dest="suggest"/>
 

data/solr/conf/solrconfig.xml

@@ -30,6 +30,7 @@
   -->
   <luceneMatchVersion>8.2.0</luceneMatchVersion>
 
+  <lib dir="${solr.install.dir:../../../..}/modules/analysis-extras/lib" />
   <lib dir="${solr.install.dir:../../../..}/contrib/analysis-extras/lib" />
   <lib dir="${solr.install.dir:../../../..}/contrib/analysis-extras/lucene-libs" />
 
@@ -87,45 +88,49 @@
        <str name="qf">
          collection_title_tesim^150
          title_tesim^100
-         name_tesim^10
-         place_tesim^10
-         unitid_identifier_match^5
-         subject_tesim^2
+         normalized_title_teim^100
+         unitid_identifier_match^40
+         name_teim^10
+         place_teim^10
+         subject_teim^2
          text
        </str>
        <str name="pf">
          collection_title_tesim^150
          title_tesim^100
-         name_tesim^10
-         place_tesim^10
-         unitid_identifier_match^5
-         subject_tesim^2
+         normalized_title_teim^100
+         unitid_identifier_match^40
+         name_teim^10
+         place_teim^10
+         subject_teim^2
          text
        </str>
 
        <str name="qf_name">
-         name_tesim
+         name_teim
        </str>
        <str name="pf_name">
-         name_tesim
+         name_teim
        </str>
        <str name="qf_place">
-         place_tesim
+         place_teim
        </str>
        <str name="pf_place">
-         place_tesim
+         place_teim
        </str>
        <str name="qf_subject">
-         subject_tesim
+         subject_teim
        </str>
        <str name="pf_subject">
-         subject_tesim
+         subject_teim
        </str>
        <str name="qf_title">
          title_tesim
+         normalized_title_teim
        </str>
        <str name="pf_title">
          title_tesim
+         normalized_title_teim
        </str>
 
        <int name="ps">3</int>
@@ -134,20 +139,18 @@
        <str name="fl">
          id,
          score,
-         abstract_ssm,
-         accessrestrict_ssm,
-         child_component_count_isim,
-         collection_ssm,
-         collection_unitid_ssm,
+         abstract_html_tesm,
+         accessrestrict_html_tesm,
+         child_component_count_isi,
          containers_ssim,
          creator_ssm,
          extent_ssm,
          geogname_ssm,
          has_online_content_ssim,
-         language_ssm,
+         language_ssim,
          level_ssm,
          repository_ssm,
-         scopecontent_ssm,
+         scopecontent_html_tesm,
          title_ssm,
          normalized_title_ssm,
          normalized_date_ssm,
@@ -157,22 +160,22 @@
          ead_ssi,
          ref_ssm,
          component_level_isim,
-         parent_access_restrict_ssm,
-         parent_access_terms_ssm,
+         parent_access_restrict_tesm,
+         parent_access_terms_tesm,
          names_coll_ssim
          places_ssim
        </str>
 
        <str name="facet">true</str>
        <str name="facet.mincount">1</str>
-       <str name="facet.field">level_sim</str>
-       <str name="facet.field">creator_sim</str>
-       <str name="facet.field">date_range_sim</str>
-       <str name="facet.field">names_sim</str>
-       <str name="facet.field">geogname_sim</str>
-       <str name="facet.field">access_subjects_sim</str>
-       <str name="facet.field">repository_sim</str>
-       <str name="facet.field">collection_sim</str>
+       <str name="facet.field">level_ssim</str>
+       <str name="facet.field">creator_ssim</str>
+       <str name="facet.field">date_range_ssim</str>
+       <str name="facet.field">names_ssim</str>
+       <str name="facet.field">geogname_ssim</str>
+       <str name="facet.field">access_subjects_ssim</str>
+       <str name="facet.field">repository_ssim</str>
+       <str name="facet.field">collection_ssim</str>
 
        <str name="spellcheck">true</str>
        <str name="spellcheck.dictionary">default</str>

data/tasks/arclight.rake

@@ -56,16 +56,17 @@ namespace :arclight do
   desc 'Seed fixture data to Solr'
   task :seed do
     puts 'Seeding index with data from spec/fixtures/ead...'
-    Dir.glob('spec/fixtures/ead/*.xml').each do |file|
-      system("FILE=#{file} rake arclight:index") # no REPOSITORY_ID
-    end
     Dir.glob('spec/fixtures/ead/*').each do |dir|
       next unless File.directory?(dir)
 
-      system("REPOSITORY_ID=#{File.basename(dir)} " \
-             'REPOSITORY_FILE=spec/fixtures/config/repositories.yml ' \
-             "DIR=#{dir} " \
-             'rake arclight:index_dir')
+      within_test_app do
+        # Sets the REPOSITORY_ID to the name of the file's containing directory
+        system("REPOSITORY_ID=#{File.basename(dir)} " \
+               "REPOSITORY_FILE=#{Arclight::Engine.root}/spec/fixtures/config/repositories.yml " \
+               "DIR=#{Arclight::Engine.root}/#{dir} " \
+               'SOLR_URL=http://127.0.0.1:8983/solr/blacklight-core ' \
+               'rake arclight:index_dir')
+      end
     end
   end
 end

data/template.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-gem 'arclight', github: 'sul-dlss/arclight'
-gem 'blacklight_range_limit', '~> 7.1'
+gem 'arclight'
 
 run 'bundle install'