base_indexer 0.4.1 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.rdoc +27 -19
- data/app/controllers/base_indexer/about_controller.rb +3 -3
- data/app/controllers/base_indexer/application_controller.rb +14 -0
- data/app/controllers/base_indexer/collections_controller.rb +47 -0
- data/app/controllers/base_indexer/items_controller.rb +6 -14
- data/config/routes.rb +3 -0
- data/lib/base_indexer/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c01b13e3646e79aa71e842073e2382e5e1a84b2
|
4
|
+
data.tar.gz: 01272b868e2e05c667e0872ceb432e17757739a5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5aae823156dd24b4b2d1ac52af7d3f2f92710610e1ba5f93fb9f012737ec3e281352c93c6900713c719051e05d615251ffbd0be1fdc447dd43fa7d02db3267b0
|
7
|
+
data.tar.gz: b2d6c804e643c4d24baf2ec4c4055e1a05c45d8b0f5e610959f299d159a4d2817fa3700e184488de9aad3143d5e2f9ef18d5f8153282b6818da8497c9a130b9a
|
data/README.rdoc
CHANGED
@@ -2,27 +2,35 @@
|
|
2
2
|
|
3
3
|
This project rocks and uses MIT-LICENSE.
|
4
4
|
|
5
|
+
== Running tests
|
6
|
+
|
7
|
+
Clone from github.
|
8
|
+
rake # first time setup and to generate all docs
|
9
|
+
bundle exec rake spec # just run the tests next time around
|
10
|
+
|
11
|
+
|
5
12
|
== Steps to hook the base_indexer engine in your app
|
6
13
|
* Generate new rails app
|
7
|
-
|
14
|
+
rails new my_indexer_app
|
8
15
|
|
9
16
|
*Edit Gemfile and add the base_indexer gem name
|
10
|
-
|
17
|
+
gem 'base_indexer'
|
11
18
|
|
12
19
|
* Run bundle install to download the gem
|
13
|
-
|
20
|
+
bundle install
|
14
21
|
|
15
22
|
* Mount the engine in your favorite domain.
|
16
|
-
|
23
|
+
mount BaseIndexer::Engine, at: '/items'
|
17
24
|
|
18
25
|
== Basic configuration
|
19
26
|
The engine is looking for the following values
|
20
27
|
|
21
|
-
|
22
|
-
|
28
|
+
config.solr_config_file_path = "#{config.root}/config/solr.yml"
|
29
|
+
DiscoveryIndexer::PURL_DEFAULT='http://purl.stanford.edu/'
|
30
|
+
|
23
31
|
|
32
|
+
== Advanced features
|
24
33
|
|
25
|
-
Advanced features
|
26
34
|
The engine gives the developer the ability to extend any of its classes
|
27
35
|
|
28
36
|
To extend any of indexer features (purl-reader, mods-reader, mapper, solr-writer)
|
@@ -44,39 +52,39 @@ All rake tasks that perform batch indexing will generate log files in the "log"
|
|
44
52
|
log file is also useful since you can pass it to the "reindexer" rake task to retry just the errored out druids. The name of the log file will depend on which
|
45
53
|
rake task you are running, and will be timestamped to be unique.
|
46
54
|
|
47
|
-
|
55
|
+
==== Index a single druid:
|
48
56
|
|
49
57
|
rake index RAILS_ENV=production target=revs_prod druid=oo000oo0001
|
50
58
|
|
51
|
-
|
59
|
+
==== Index a list of druids from a pre-assembly run, a remeditaion run, or a simple CSV:
|
52
60
|
|
53
|
-
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly
|
54
|
-
nohup rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly &
|
61
|
+
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly = preassembly run
|
62
|
+
nohup rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly & = for a long running process, which will be most runs that have more than a few dozen druids, nohup it
|
55
63
|
|
56
|
-
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1_remediate.yaml log_type=remediate
|
64
|
+
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1_remediate.yaml log_type=remediate = remediation run
|
57
65
|
|
58
|
-
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander.csv log_type=csv
|
66
|
+
rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander.csv log_type=csv = a simple csv file -- it must have a header line, with the header of "druid" definining the items you wish to index
|
59
67
|
|
60
|
-
|
68
|
+
==== Index an entire collection, including the collection itself, along with all of its members (be sure to check the dor-fetcher-url parameter in the Rails environment you are running under to be sure it is connecting where you expect):
|
61
69
|
|
62
70
|
rake collection_indexer RAILS_ENV=production target=revs_prod collection_druid=oo000oo0001
|
63
|
-
nohup rake collection_indexer RAILS_ENV=production target=revs_prod collection_druid=oo000oo0001 &
|
71
|
+
nohup rake collection_indexer RAILS_ENV=production target=revs_prod collection_druid=oo000oo0001 & = for a long running process, e.g. a collection with more than a few dozen druids, nohup it
|
64
72
|
|
65
|
-
|
73
|
+
==== Re-Index Just Errored Out Items
|
66
74
|
|
67
75
|
If you had errors when indexing from a preassembly/remediation log or from indexing an entire collection, you can re-run the errored out druids only with the log file. All log files are kept in the log folder in the revs-indexer-service app.
|
68
76
|
|
69
77
|
rake reindexer RAILS_ENV=production target=revs_prod file=log/logfile.log
|
70
78
|
|
71
|
-
nohup rake reindexer RAILS_ENV=production target=revs_prod file=log/logfile.log &
|
79
|
+
nohup rake reindexer RAILS_ENV=production target=revs_prod file=log/logfile.log & = probably no need to nohup unless there were alot of errors
|
72
80
|
|
73
81
|
|
74
|
-
|
82
|
+
==== Delete Druids
|
75
83
|
|
76
84
|
Delete a list of druids specified in a CSV/txt file. Be careful, this will delete from all targets! Put one druid per line, no header is necessary.
|
77
85
|
|
78
86
|
rake delete_druids RAILS_ENV=production file=druid_list.txt
|
79
87
|
|
80
|
-
Delete a single druid
|
88
|
+
==== Delete a single druid
|
81
89
|
|
82
90
|
rake delete RAILS_ENV=production druid=oo000oo0001
|
@@ -2,7 +2,7 @@ module BaseIndexer
|
|
2
2
|
class AboutController < ApplicationController
|
3
3
|
|
4
4
|
def index
|
5
|
-
render :
|
5
|
+
render text: 'ok', status: 200
|
6
6
|
end
|
7
7
|
|
8
8
|
def version
|
@@ -11,8 +11,8 @@ module BaseIndexer
|
|
11
11
|
@result.update({:solr_cores=>BaseIndexer.solr_configuration_class_name.constantize.instance.get_configuration_hash})
|
12
12
|
|
13
13
|
respond_to do |format|
|
14
|
-
format.json {render :
|
15
|
-
format.xml {render :
|
14
|
+
format.json {render json: @result.to_json}
|
15
|
+
format.xml {render json: result.to_xml(root: 'status')}
|
16
16
|
format.html {render}
|
17
17
|
# add the solr core names
|
18
18
|
end
|
@@ -1,5 +1,19 @@
|
|
1
1
|
module BaseIndexer
|
2
2
|
class ApplicationController < ActionController::Base
|
3
3
|
|
4
|
+
respond_to :json, :xml, :html
|
5
|
+
|
6
|
+
def remove_prefix druid
|
7
|
+
druid.gsub('druid:','') # lop off druid prefix if sent
|
8
|
+
end
|
9
|
+
|
10
|
+
def report_failure method_symbol, params, e
|
11
|
+
return "#{method_symbol} #{params}\n\n#{e.inspect}\n#{e.message}\n#{e.backtrace}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def report_success
|
15
|
+
return "success"
|
16
|
+
end
|
17
|
+
|
4
18
|
end
|
5
19
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require_dependency "base_indexer/application_controller"
|
2
|
+
|
3
|
+
module BaseIndexer
|
4
|
+
class CollectionsController < ApplicationController
|
5
|
+
|
6
|
+
def new
|
7
|
+
begin
|
8
|
+
druid = remove_prefix params[:druid]
|
9
|
+
|
10
|
+
Rails.logger.debug "Receiving indexing of collection #{druid}"
|
11
|
+
targets = params[:subtargets]
|
12
|
+
|
13
|
+
# initial collection item itself
|
14
|
+
indexer = BaseIndexer.indexer_class.constantize.new
|
15
|
+
indexer.index druid, targets
|
16
|
+
|
17
|
+
# initialize dor-fetcher to get list of druids for this collection
|
18
|
+
df = DorFetcher::Client.new({:service_url => Rails.application.config.dor_fetcher_url})
|
19
|
+
|
20
|
+
item_druids = df.druid_array(df.get_collection(druid, {}))
|
21
|
+
|
22
|
+
Rails.logger.debug "Found #{item_druids.size} members of the collection #{druid}"
|
23
|
+
|
24
|
+
counter=0
|
25
|
+
|
26
|
+
item_druids.each do |druid|
|
27
|
+
|
28
|
+
druid=remove_prefix druid
|
29
|
+
counter+=1
|
30
|
+
indexer.index druid,targets
|
31
|
+
Rails.logger.debug "#{counter} of #{item_druids.size}: #{druid}"
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
@status = report_success
|
36
|
+
render nothing: true, status: 200
|
37
|
+
Rails.logger.debug "Completing indexing of collection #{druid}"
|
38
|
+
|
39
|
+
rescue Exception => e
|
40
|
+
@status = report_failure request.method_symbol, params, e
|
41
|
+
Rails.logger.error @status
|
42
|
+
render nothing: true, status: 202
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -2,49 +2,41 @@ require_dependency "base_indexer/application_controller"
|
|
2
2
|
|
3
3
|
module BaseIndexer
|
4
4
|
class ItemsController < ApplicationController
|
5
|
-
respond_to :json
|
6
5
|
|
7
6
|
def new
|
8
7
|
begin
|
9
|
-
druid = params[:druid]
|
8
|
+
druid = remove_prefix params[:druid]
|
10
9
|
Rails.logger.debug "Receiving indexing #{druid}"
|
11
10
|
targets = params[:subtargets]
|
12
11
|
|
13
12
|
indexer = BaseIndexer.indexer_class.constantize.new
|
14
13
|
indexer.index druid, targets
|
15
14
|
@status = report_success
|
16
|
-
render status: 200
|
15
|
+
render nothing: true, status: 200
|
17
16
|
Rails.logger.debug "Completing indexing #{druid}"
|
18
17
|
rescue Exception => e
|
19
18
|
@status = report_failure request.method_symbol, params, e
|
20
19
|
Rails.logger.error @status
|
21
|
-
render status: 202
|
20
|
+
render nothing: true, status: 202
|
22
21
|
end
|
23
22
|
end
|
24
23
|
|
25
24
|
def destroy
|
26
25
|
begin
|
27
|
-
druid = params[:druid]
|
26
|
+
druid = remove_prefix params[:druid]
|
28
27
|
Rails.logger.debug "Receiving deleting #{druid}"
|
29
28
|
|
30
29
|
indexer = BaseIndexer.indexer_class.constantize.new
|
31
30
|
indexer.delete druid
|
32
31
|
@status= report_success
|
33
|
-
render status: 200
|
32
|
+
render nothing: true, status: 200
|
34
33
|
Rails.logger.debug "Completing deleting #{druid}"
|
35
34
|
rescue Exception => e
|
36
35
|
@status = report_failure request.method_symbol, params, e
|
37
36
|
Rails.logger.error @status
|
38
|
-
render status: 202
|
37
|
+
render nothing: true, status: 202
|
39
38
|
end
|
40
39
|
end
|
41
40
|
|
42
|
-
def report_failure method_symbol, params, e
|
43
|
-
return "#{method_symbol} #{params}\n\n#{e.inspect}\n#{e.message}\n#{e.backtrace}"
|
44
|
-
end
|
45
|
-
|
46
|
-
def report_success
|
47
|
-
return "success"
|
48
|
-
end
|
49
41
|
end
|
50
42
|
end
|
data/config/routes.rb
CHANGED
data/lib/base_indexer/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: base_indexer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ahmed Alsum
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-06-
|
11
|
+
date: 2015-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -223,6 +223,7 @@ files:
|
|
223
223
|
- Rakefile
|
224
224
|
- app/controllers/base_indexer/about_controller.rb
|
225
225
|
- app/controllers/base_indexer/application_controller.rb
|
226
|
+
- app/controllers/base_indexer/collections_controller.rb
|
226
227
|
- app/controllers/base_indexer/items_controller.rb
|
227
228
|
- app/helpers/base_indexer/application_helper.rb
|
228
229
|
- app/helpers/base_indexer/items_helper.rb
|