spotlight-atom 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bdbbd2763c5600927de41dacadcb558ade4e3b27
+  data.tar.gz: 8012d2d20656bd4c4fe716091ca189dee1fac779
+SHA512:
+  metadata.gz: d453b5be60212c19aa09fbe80dd5ed24404684d6d10a6643cf4815a0efdf340f317e32f9e74099d328dc13fd57042151923fa5505004d7f99510cbac1500f79d
+  data.tar.gz: 86a17c6c78a85bbcc0d73203a8574ec07ed499954dfcf7cfd81a5af73a2be05d3f35487682f8d77f171b3dd0802f8be524f71fc254e0149718a8aa0a4c15afc9

data/.gitignore

@@ -0,0 +1,22 @@
+*.rbc
+*.sassc
+.sass-cache
+capybara-*.html
+.rspec
+.rvmrc
+/.bundle
+/vendor/bundle
+/log/*
+/tmp/*
+/db/*.sqlite3
+/public/system/*
+/coverage/
+/spec/tmp/*
+**.orig
+rerun.txt
+pickle-email-*.html
+.project
+config/initializers/secret_token.rb
+spec/internal
+Gemfile.lock
+jetty

data/.travis.yml

@@ -0,0 +1,18 @@
+notifications:
+  email: false
+
+rvm:
+  - 2.1
+
+notifications:
+  irc: "irc.freenode.org#blacklight"
+  email:
+      - exhibits-team@lists.stanford.edu
+
+env:
+  global:
+    - JRUBY_OPTS="-J-Xms512m -J-Xmx1024m"
+    - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+
+sudo: false
+language: ruby

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spotlight-atom-resources.gemspec
+gemspec

data/README.md

@@ -0,0 +1,31 @@
+# Spotlight::Atom
+
+Harvest resources in Atom feeds into a Spotlight exhibit
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'spotlight-atom'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install spotlight-atom
+
+## Usage
+
+This gem adds a new  "Repository Item" form to your Spotlight application. This form allows curators to input a URL to an Atom feed, and the contents of the feed will be harvested as new items in the Spotlight exhibit. Currently, this gem supports harvesting basic metadata from either Blacklight or Omeka feeds.
+
+## Contributing
+
+1. Fork it ( https://github.com/cbeer/spotlight-atom/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,45 @@
+require "bundler/gem_tasks"
+
+ZIP_URL = "https://github.com/projectblacklight/blacklight-jetty/archive/v4.10.4.zip"
+
+require 'jettywrapper'
+
+require 'engine_cart/rake_task'
+EngineCart.fingerprint_proc = EngineCart.rails_fingerprint_proc
+
+task default: :ci
+
+task :ci => ['engine_cart:generate', 'jetty:clean', 'configure_jetty'] do
+  ENV['environment'] = "test"
+  jetty_params = Jettywrapper.load_config
+  jetty_params[:startup_wait]= 60
+
+  Jettywrapper.wrap(jetty_params) do
+    # run the tests
+    Rake::Task["spec"].invoke
+  end
+end
+
+task :configure_jetty do
+  FileList['solr_conf/conf/*'].each do |f|  
+    cp("#{f}", 'jetty/solr/blacklight-core/conf/', :verbose => true)
+  end
+end
+
+task :server do
+  Rake::Task['engine_cart:generate'].invoke
+
+  unless File.exists? 'jetty'
+    Rake::Task['jetty:clean'].invoke
+    Rake::Task['configure_jetty'].invoke
+  end
+
+  jetty_params = Jettywrapper.load_config
+  jetty_params[:startup_wait]= 60
+
+  Jettywrapper.wrap(jetty_params) do
+    within_test_app do
+      system "bundle exec rails s"
+    end
+  end
+end

data/app/models/spotlight/resources/atom_harvester.rb

@@ -0,0 +1,96 @@
+module Spotlight::Resources
+  class AtomHarvester < Spotlight::Resource
+    self.weight = -5000
+
+    after_save :harvest_resources
+    
+    def self.can_provide? res
+      is_atom?(res.url) || !!(res.url =~ /.atom/)
+    end
+
+    def self.is_atom? url
+      Faraday.head(url) do |req|
+        req.headers['Accept'] = 'application/atom+xml'
+      end.status == 200
+    end
+   
+    def update_index data
+      data = [data] unless data.is_a? Array
+      blacklight_solr.update params: { commitWithin: 500 }, data: data.to_json, headers: { 'Content-Type' => 'application/json'} unless data.empty?
+    end
+   
+    def to_solr
+      []
+    end
+
+    def harvest_resources
+      items.each do |x|
+        h = convert_entry_to_solr_hash(x)
+        puts "creating #{h.inspect}"
+        Spotlight::Resources::Upload.create(
+          remote_url_url: h[:url],
+          data: h,
+          exhibit: exhibit
+        ) if h[:url]
+      end
+    end
+
+    def items
+      return enum_for(:items) unless block_given?
+
+      doc = fetch(self.url)
+      while doc and doc.xpath('//atom:entry', ns).any?
+        h = doc.xpath('//atom:entry', ns).map do |x|
+          yield x
+        end
+
+        if doc.xpath('/atom:feed/atom:link[@rel="next"]', ns).any?
+          doc = fetch(doc.xpath('/atom:feed/atom:link[@rel="next"]', ns).first.attr('href'))
+        else
+          doc = nil
+          break
+        end
+      end
+    end
+
+    def convert_entry_to_solr_hash x  
+      { 
+        exhibit.blacklight_config.document_model.unique_key.to_sym => compound_id(x),
+        title_field => x.xpath('atom:title', ns).text, 
+        url: x.xpath('atom:link[contains(@rel, "enclosure")]/@href', ns).text
+      }
+    end
+
+    def compound_id x
+      x.xpath("atom:id", ns).text.parameterize
+    end
+
+    def fetch url
+      Nokogiri::XML(Faraday.get(url) do |req|
+        req.headers['Accept'] = 'application/atom+xml'
+      end.body)
+    end
+
+    def ns
+      { atom: "http://www.w3.org/2005/Atom" }
+    end
+
+    def title_field
+      Spotlight::Engine.config.upload_title_field || exhibit.blacklight_config.index.title_field
+    end
+    
+    def create_sidecars_for *keys
+      missing = keys - exhibit.custom_fields.map { |x| x.label }
+
+      missing.each do |k|
+        exhibit.custom_fields.create! label: k
+      end.tap { @exhibit_custom_fields = nil }
+    end
+
+    def exhibit_custom_fields
+      @exhibit_custom_fields ||= exhibit.custom_fields.each_with_object({}) do |value, hash|
+        hash[value.label] = value
+      end
+    end
+  end
+end

data/app/models/spotlight/resources/blacklight_harvester.rb

@@ -0,0 +1,21 @@
+module Spotlight::Resources
+  class BlacklightHarvester < Spotlight::Resources::AtomHarvester
+    def convert_entry_to_solr_hash x
+      content = atom_content_to_hash(x)
+      create_sidecars_for(*content.keys)
+      h = {}
+      
+      content.each_with_object(h) do |(key, value), hash|
+        h[exhibit_custom_fields[key].field] = value
+      end
+
+      super.merge h
+    end
+
+    def atom_content_to_hash x
+      content = Hash[Nokogiri::HTML(x.xpath("atom:content", ns).text).css('dt').map do |dt|
+        [dt.text, dt.first.next('dd').text] 
+      end].reject { |k,v| k.blank? }
+    end
+  end
+end

data/app/models/spotlight/resources/omeka_harvester.rb

@@ -0,0 +1,22 @@
+module Spotlight::Resources
+  class OmekaHarvester < Spotlight::Resources::AtomHarvester
+    def convert_entry_to_solr_hash x
+      content = atom_content_to_hash(x)
+      create_sidecars_for(*content.keys)
+      h = {}
+      
+      content.each_with_object(h) do |(key, value), hash|
+        h[exhibit_custom_fields[key].field] = value
+      end
+
+      super.merge h
+    end
+
+    def atom_content_to_hash x
+      content = Hash[Nokogiri::HTML(x.xpath("atom:content", ns).text).css('.element').map do |d| 
+        [d.css('h3').text, d.css('.element-text').text] 
+      end].reject { |k,v| k.blank? }
+    end
+
+  end
+end

data/app/views/spotlight/resources/atom/_blacklight.html.erb

@@ -0,0 +1,9 @@
+<%= bootstrap_form_for([current_exhibit, @resource], layout: :horizontal, label_col: 'col-md-2', control_col: 'col-sm-6 col-md-6', html: { class: 'item-upload-form', multipart: true } ) do |f| %>
+  <%= f.text_area :url, help: t('.url-field.help') %>
+  <div class="form-actions">
+    <div class="primary-actions">
+      <%= cancel_link @resource, :back, class: 'btn btn-default' %>
+      <%= f.submit t('.add_item'), class: 'btn btn-primary' %>
+    </div>
+  </div>
+<% end %>

data/app/views/spotlight/resources/atom/_omeka.html.erb

@@ -0,0 +1,9 @@
+<%= bootstrap_form_for([current_exhibit, @resource], layout: :horizontal, label_col: 'col-md-2', control_col: 'col-sm-6 col-md-6', html: { class: 'item-upload-form', multipart: true } ) do |f| %>
+  <%= f.text_area :url, help: t('.url-field.help') %>
+  <div class="form-actions">
+    <div class="primary-actions">
+      <%= cancel_link @resource, :back, class: 'btn btn-default' %>
+      <%= f.submit t('.add_item'), class: 'btn btn-primary' %>
+    </div>
+  </div>
+<% end %>

data/app/views/spotlight/resources/atom/_tabbed_form.html.erb

@@ -0,0 +1,18 @@
+<div role="tabpanel" class="item-upload-tabs">
+  <ul class="nav nav-tabs" role="tablist">
+    <% Spotlight::Atom::Resources::Engine.config.resource_partials.each_with_index do |p, i| %>
+      <% name = p.split('/').last %>
+      <li role="presentation" class="<%= "active" if i ==0 %>">
+        <%= link_to t(".#{name}"), "##{name}", role: 'tab', 'data-toggle' => 'tab', 'aria-controls' => 'single' %>
+      </li>
+    <% end %>
+  </ul>
+  <div class="tab-content">
+    <% Spotlight::Atom::Resources::Engine.config.resource_partials.each_with_index do |p, i| %>
+      <% name = p.split('/').last %>
+      <%= content_tag :div, id: name, role: 'tabpanel', class: "tab-pane #{"active" if i == 0}" do %>
+        <%= render p %>
+      <% end %>
+    <% end %>
+  </div>
+</div>

data/config/locales/spotlight-atom-resources.en.yml

@@ -0,0 +1,11 @@
+en:
+  spotlight:
+    resources:
+      atom:
+        tabbed_form:
+          omeka: "Omeka"
+          blacklight: "Blacklight"
+        blacklight:
+          add_item: "Import feed"
+        omeka:
+          add_item: "Import feed"

data/lib/spotlight/atom.rb

@@ -0,0 +1,9 @@
+require 'spotlight/atom/version'
+require 'spotlight'
+require 'faraday'
+
+module Spotlight
+  module Atom
+    require 'spotlight/atom/engine'
+  end
+end

data/lib/spotlight/atom/engine.rb

@@ -0,0 +1,16 @@
+require 'spotlight/engine'
+
+module Spotlight
+  module Atom
+    class Engine < ::Rails::Engine
+      Spotlight::Resources::Engine.config.resource_partials = ['spotlight/resources/atom/blacklight', 'spotlight/resources/atom/omeka']
+      initializer "spotlight.dor.initialize" do
+        Spotlight::Engine.config.resource_providers << Spotlight::Resources::OmekaHarvester
+        Spotlight::Engine.config.resource_providers << Spotlight::Resources::BlacklightHarvester
+        Spotlight::Engine.config.resource_providers << Spotlight::Resources::AtomHarvester
+        Spotlight::Engine.config.new_resource_partials ||= []
+        Spotlight::Engine.config.new_resource_partials << 'spotlight/resources/atom/tabbed_form'
+      end
+    end
+  end
+end

data/lib/spotlight/atom/version.rb

@@ -0,0 +1,5 @@
+module Spotlight
+  module Atom
+    VERSION = "0.0.1"
+  end
+end

data/solr_conf/conf/schema.xml

@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema name="Hydra" version="1.5">
+  <!-- NOTE: various comments and unused configuration possibilities have been purged
+     from this file.  Please refer to http://wiki.apache.org/solr/SchemaXml,
+     as well as the default schema file included with Solr -->
+  
+  <uniqueKey>id</uniqueKey>
+  
+  <fields>
+    <field name="id" type="string" stored="true" indexed="true" multiValued="false" required="true"/>
+    <field name="_version_" type="long"     indexed="true"  stored="true"/>
+    <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+    <dynamicField name="*_bbox"   type="location_rpt" stored="true" indexed="true"/>
+    <dynamicField name="*_ng"   type="text_en_ng" stored="false" indexed="true" multiValued="true"/>
+
+    <dynamicField name="*_ts" type="text" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_tsm" type="text" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_tsi" type="text" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_tsim" type="text" stored="true" indexed="true" multiValued="true"/>
+    <dynamicField name="*_tsiv" type="text" stored="true" indexed="true" multiValued="false" termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tsimv" type="text" stored="true" indexed="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/>
+
+    <dynamicField name="*_tes" type="text_en" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_tesm" type="text_en" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_tesi" type="text_en" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_tesim" type="text_en" stored="true" indexed="true" multiValued="true"/>
+    <dynamicField name="*_tesiv" type="text_en" stored="true" indexed="true" multiValued="false" termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tesimv" type="text_en" stored="true" indexed="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/>
+
+    <dynamicField name="*_ss" type="string" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_ssm" type="string" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_ssi" type="string" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_ssim" type="string" stored="true" indexed="true" multiValued="true"/>
+
+    <dynamicField name="*_is" type="int" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_ism" type="int" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_isi" type="int" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_isim" type="int" stored="true" indexed="true" multiValued="true"/>
+
+    <dynamicField name="*_dts" type="date" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_dtsm" type="date" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_dtsi" type="date" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_dtsim" type="date" stored="true" indexed="true" multiValued="true"/>
+    
+    <dynamicField name="*_ls" type="long" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_lsm" type="long" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_lsi" type="long" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_lsim" type="long" stored="true" indexed="true" multiValued="true"/>
+    
+    <!-- double (_db...) -->
+    <dynamicField name="*_dbs" type="double" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_dbsm" type="double" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_dbsi" type="double" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_dbsim" type="double" stored="true" indexed="true" multiValued="true"/>
+
+    <dynamicField name="*_fs" type="float" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_fsm" type="float" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_fsi" type="float" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_fsim" type="float" stored="true" indexed="true" multiValued="true"/>
+
+    <dynamicField name="*_bs" type="boolean" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_bsi" type="boolean" stored="true" indexed="true" multiValued="false"/>
+
+    <dynamicField name="*_lls" type="location" stored="true" indexed="false" multiValued="false"/>
+    <dynamicField name="*_llsm" type="location" stored="true" indexed="false" multiValued="true"/>
+    <dynamicField name="*_llsi" type="location" stored="true" indexed="true" multiValued="false"/>
+    <dynamicField name="*_llsim" type="location" stored="true" indexed="true" multiValued="true"/>
+
+    <!-- you must define copyField source and dest fields explicity or schemaBrowser doesn't work -->
+    <field name="all_text_timv" type="text" stored="false" indexed="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/>
+
+
+  </fields>
+
+
+  <!-- Above, multiple source fields are copied to the [text] field.
+      Another way to map multiple source fields to the same
+      destination field is to use the dynamic field syntax.
+      copyField also supports a maxChars to copy setting.  -->
+
+  <copyField source="id" dest="id_ng" maxChars="3000"/>
+  <copyField source="full_title_tesim" dest="full_title_ng" maxChars="3000"/>
+  <copyField source="*_tesim" dest="all_text_timv" maxChars="3000"/>
+  <copyField source="*_ssim" dest="all_text_timv" maxChars="3000"/>
+
+  <types>
+    <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>    
+    <fieldType name="rand" class="solr.RandomSortField" omitNorms="true"/>
+    
+    <!-- Default numeric field types.  -->
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+    
+    <!-- The format for this date field is of the form 1995-12-31T23:59:59Z
+         Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+      -->
+    <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+    
+    <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
+    <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+    
+    <!-- An alternative geospatial field type new to Solr 4.  It supports multiValued and polygon shapes.
+      For more information about this and other Spatial fields new to Solr 4, see:
+      http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
+    -->
+    <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
+      geo="true" distErrPct="0.025" maxDistErr="0.000009" units="degrees" />
+    
+    <fieldType name="text" class="solr.TextField" omitNorms="false">
+      <analyzer>
+        <tokenizer class="solr.ICUTokenizerFactory"/>
+        <filter class="solr.ICUFoldingFilterFactory"/>  <!-- NFKC, case folding, diacritics removed -->
+        <filter class="solr.TrimFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- A text field with defaults appropriate for English -->
+    <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+      <analyzer>
+        <tokenizer class="solr.ICUTokenizerFactory"/>
+        <filter class="solr.ICUFoldingFilterFactory"/>  <!-- NFKC, case folding, diacritics removed -->
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <!-- EnglishMinimalStemFilterFactory is less aggressive than PorterStemFilterFactory: -->
+        <filter class="solr.EnglishMinimalStemFilterFactory"/>
+        <!--
+        <filter class="solr.PorterStemFilterFactory"/>
+        -->
+        <filter class="solr.TrimFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- A text field with defaults appropriate for English an NGrams -->
+    <fieldType name="text_en_ng" class="solr.TextField" positionIncrementGap="100">
+      <analyzer type="index">
+        <tokenizer class="solr.ICUTokenizerFactory"/>
+        <filter class="solr.ICUFoldingFilterFactory"/>  <!-- NFKC, case folding, diacritics removed -->
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <!-- EnglishMinimalStemFilterFactory is less aggressive than PorterStemFilterFactory: -->
+        <filter class="solr.EnglishMinimalStemFilterFactory"/>
+        <filter class="solr.TrimFilterFactory"/>
+        <filter class="solr.EdgeNGramFilterFactory" minGramSize="3" maxGramSize="15" />
+      </analyzer>
+
+      <analyzer type="index">
+        <tokenizer class="solr.ICUTokenizerFactory"/>
+        <filter class="solr.ICUFoldingFilterFactory"/>  <!-- NFKC, case folding, diacritics removed -->
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <!-- EnglishMinimalStemFilterFactory is less aggressive than PorterStemFilterFactory: -->
+        <filter class="solr.EnglishMinimalStemFilterFactory"/>
+        <filter class="solr.TrimFilterFactory"/>
+      </analyzer>
+    </fieldType>
+  </types>
+  
+</schema>

data/solr_conf/conf/solrconfig.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<config>
+  <!-- NOTE: various comments and unused configuration possibilities have been purged
+     from this file.  Please refer to http://wiki.apache.org/solr/SolrConfigXml,
+     as well as the default solrconfig file included with Solr -->
+  
+  <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
+  
+  <luceneMatchVersion>4.10.3</luceneMatchVersion>
+
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.core0.data.dir:}</str>
+    </updateLog>
+  </updateHandler>
+
+  <!-- solr lib dirs -->
+  <lib dir="../lib/contrib/analysis-extras/lib" />
+  <lib dir="../lib/contrib/analysis-extras/lucene-libs" />
+  
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <requestHandler name="search" class="solr.SearchHandler" default="true">
+     <lst name="defaults">
+       <str name="defType">edismax</str>
+       <str name="echoParams">explicit</str>
+       <str name="q.alt">*:*</str>
+       <str name="mm">2&lt;-1 5&lt;-2 6&lt;90%</str>
+       <int name="qs">1</int>
+       <int name="ps">2</int>
+       <float name="tie">0.01</float>
+        <str name="qf">
+          id^1000
+          full_title_tesim^100
+          spotlight_upload_description_tesim^50
+          all_text_timv
+        </str>
+        <str name="pf">
+          full_title_tesim^100
+          all_text_timv^10
+        </str>
+
+       <str name="fl">
+         *, score
+       </str>
+
+       <str name="facet">true</str>
+       <str name="facet.mincount">1</str>
+       <str name="facet.limit">10</str>
+     </lst>
+  </requestHandler>
+
+  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+    <lst name="defaults">
+      <str name="omitHeader">true</str>
+      <str name="wt">json</str>
+      <str name="indent">true</str>
+    </lst>
+  </requestHandler>
+
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
+  
+  <requestDispatcher handleSelect="true" >
+    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
+  </requestDispatcher>
+  
+  <requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
+  <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+  
+  <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+    <lst name="invariants">
+      <str name="q">solrpingquery</str>
+    </lst>
+    <lst name="defaults">
+      <str name="echoParams">all</str>
+    </lst>
+  </requestHandler>
+  
+  <!-- config for the admin interface --> 
+  <admin>
+    <defaultQuery>search</defaultQuery>
+  </admin>
+  
+</config>

data/spec/spec_helper.rb

@@ -0,0 +1,83 @@
+ENV["RAILS_ENV"] ||= 'test'
+
+require 'factory_girl'
+require 'database_cleaner'
+require 'devise'
+require 'engine_cart'
+EngineCart.load_application!
+
+require 'rspec/rails'
+
+require 'capybara/poltergeist'
+
+if ENV["POLTERGEIST_DEBUG"]
+  Capybara.register_driver :poltergeist_debug do |app|
+    Capybara::Poltergeist::Driver.new(app, inspector: true, phantomjs_options: ['--load-images=no'])
+  end
+  Capybara.javascript_driver = :poltergeist_debug
+else
+  Capybara.register_driver :poltergeist do |app|
+    Capybara::Poltergeist::Driver.new(app, phantomjs_options: ['--load-images=no'])
+  end
+  Capybara.javascript_driver = :poltergeist
+end
+Capybara.default_wait_time = 10
+
+require 'spotlight/atom/resources'
+
+
+Dir["./spec/support/**/*.rb"].sort.each {|f| require f}
+
+FactoryGirl.definition_file_paths = [File.expand_path("../factories", __FILE__)]
+FactoryGirl.find_definitions
+
+FIXTURES_PATH = File.expand_path("../fixtures", __FILE__);
+
+
+RSpec.configure do |config|
+  config.infer_spec_type_from_file_location!
+
+  config.use_transactional_fixtures = false
+
+  config.before :each do
+    if Capybara.current_driver == :rack_test
+      DatabaseCleaner.strategy = :transaction
+    else
+      DatabaseCleaner.strategy = :truncation
+    end
+    DatabaseCleaner.start
+
+    # The first user is automatically granted admin privileges; we don't want that behavior for many of our tests
+    User.create email: 'initial+admin@example.com', password: 'password', password_confirmation: 'password'
+  end
+
+  config.after do
+    DatabaseCleaner.clean
+  end
+
+  if ENV['CI']
+    config.filter_run_excluding js: true
+  end
+
+  config.include Devise::TestHelpers, type: :controller
+  config.include Devise::TestHelpers, type: :view
+  config.include Spotlight::TestViewHelpers, type: :view
+  config.include Warden::Test::Helpers, type: :feature
+  config.after(:each, type: :feature) { Warden.test_reset! }
+  config.include Controllers::EngineHelpers, type: :controller
+  config.include Capybara::DSL
+  config.include Spotlight::TestFeaturesHelpers, type: :feature
+end
+
+def add_new_page_via_button(title="New Page")
+  add_link = find("[data-expanded-add-button]")
+  within(add_link) do
+    expect(page).to have_css("input[type='text']", visible: false)
+  end
+  add_link.hover
+  within(add_link) do
+    input = find("input[type='text']", visible: true)
+    input.set(title)
+    find("input[data-behavior='save']").click
+  end
+end

data/spec/test_app_templates/Gemfile.extra

@@ -0,0 +1 @@
+gem "sir_trevor_rails", github: "sul-dlss/sir-trevor-rails"

data/spec/test_app_templates/carrierwave.rb

@@ -0,0 +1,6 @@
+if Rails.env.test? or Rails.env.cucumber?
+  CarrierWave.configure do |config|
+    config.storage = :file
+    config.enable_processing = false
+  end
+end

data/spec/test_app_templates/lib/generators/test_app_generator.rb

@@ -0,0 +1,30 @@
+require 'rails/generators'
+
+class TestAppGenerator < Rails::Generators::Base
+  source_root "../../spec/test_app_templates"
+
+  def add_gems
+    gem 'blacklight', ">= 5.4.0.rc1", "<6"
+    gem "blacklight-gallery", ">= 0.3.0"
+    gem "sir_trevor_rails", github: "sul-dlss/sir-trevor-rails"
+    gem "blacklight-spotlight", github: "sul-dlss/spotlight"
+    gem "jettywrapper"
+    Bundler.with_clean_env do
+      run "bundle install"
+    end
+  end
+
+  def run_blacklight_generator
+    generate 'blacklight:install', '--devise'
+  end
+  
+  def add_spotlight_routes_and_assets
+    generate 'spotlight:install', '-f --mailer_default_url_host=localhost:3000'
+  end
+
+  def run_spotlight_migrations
+    rake "spotlight:install:migrations"
+    rake "db:migrate"
+  end
+
+end

data/spotlight-atom.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spotlight/atom/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "spotlight-atom"
+  spec.version       = Spotlight::Atom::VERSION
+  spec.authors       = ["Chris Beer", "Jessie Keck"]
+  spec.email         = ["chris@cbeer.info", "jkeck@stanford.edu"]
+  spec.summary       = %q{Harvesting atom feeds into Spotlight}
+  spec.homepage      = ""
+  spec.license       = "Apache 2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "blacklight-spotlight"
+  spec.add_dependency "faraday"
+  
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "rspec-rails", "~> 3.1"
+  spec.add_development_dependency "capybara"
+  spec.add_development_dependency "poltergeist", ">= 1.5.0"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "engine_cart"
+  spec.add_development_dependency "jettywrapper"
+end

metadata

@@ -0,0 +1,212 @@
+--- !ruby/object:Gem::Specification
+name: spotlight-atom
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Chris Beer
+- Jessie Keck
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: blacklight-spotlight
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: engine_cart
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jettywrapper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- chris@cbeer.info
+- jkeck@stanford.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- app/models/spotlight/resources/atom_harvester.rb
+- app/models/spotlight/resources/blacklight_harvester.rb
+- app/models/spotlight/resources/omeka_harvester.rb
+- app/views/spotlight/resources/atom/_blacklight.html.erb
+- app/views/spotlight/resources/atom/_omeka.html.erb
+- app/views/spotlight/resources/atom/_tabbed_form.html.erb
+- config/locales/spotlight-atom-resources.en.yml
+- lib/spotlight/atom.rb
+- lib/spotlight/atom/engine.rb
+- lib/spotlight/atom/version.rb
+- solr_conf/conf/schema.xml
+- solr_conf/conf/solrconfig.xml
+- spec/spec_helper.rb
+- spec/test_app_templates/Gemfile.extra
+- spec/test_app_templates/carrierwave.rb
+- spec/test_app_templates/lib/generators/test_app_generator.rb
+- spotlight-atom.gemspec
+homepage: ''
+licenses:
+- Apache 2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Harvesting atom feeds into Spotlight
+test_files:
+- spec/spec_helper.rb
+- spec/test_app_templates/Gemfile.extra
+- spec/test_app_templates/carrierwave.rb
+- spec/test_app_templates/lib/generators/test_app_generator.rb