base_indexer 0.4.8 → 0.4.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +8 -11
- data/app/controllers/base_indexer/about_controller.rb +10 -14
- data/app/controllers/base_indexer/application_controller.rb +7 -9
- data/app/controllers/base_indexer/collections_controller.rb +32 -38
- data/app/controllers/base_indexer/items_controller.rb +27 -33
- data/config/initializers/base_indexer.rb +6 -6
- data/config/initializers/is_it_working.rb +1 -2
- data/config/routes.rb +0 -1
- data/lib/base_indexer.rb +5 -6
- data/lib/base_indexer/engine.rb +11 -13
- data/lib/base_indexer/version.rb +1 -1
- data/lib/generators/base_indexer/install_generator.rb +6 -7
- metadata +4 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fafea5ec9ef440680cfc8a75fc6baa0ae9e976f3
|
4
|
+
data.tar.gz: 5bb1d6677f86ab0956b2bc8d32258f5e8f0550fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1c76f5a3f5ffa68ac9490f86696c69020993e6254c6aedddca60731f9d4debb9fdf2eac1129d68e9fa89a1335c34d5371c24975b60bf9f633ff82ca396cf0983
|
7
|
+
data.tar.gz: 37181e9abc7c82f855bdeba95272b3d6153ec96eab2832f8f842abce19eb702fcad6a9e2af93e41a8b44d503d36258c7916e044c42200ea7cd2e7b6ed3daa8ed
|
data/Rakefile
CHANGED
@@ -11,16 +11,16 @@ require 'yard/rake/yardoc_task'
|
|
11
11
|
|
12
12
|
Bundler::GemHelper.install_tasks
|
13
13
|
|
14
|
-
Dir[File.join(File.dirname(__FILE__), 'tasks/**/*.rake')].each {|f| load f }
|
14
|
+
Dir[File.join(File.dirname(__FILE__), 'tasks/**/*.rake')].each { |f| load f }
|
15
15
|
|
16
16
|
# Travis ci task
|
17
|
-
task :
|
18
|
-
desc
|
17
|
+
task default: :ci
|
18
|
+
desc 'run continuous integration suite (tests, coverage, docs)'
|
19
19
|
|
20
20
|
require 'engine_cart/rake_task'
|
21
21
|
EngineCart.fingerprint_proc = EngineCart.rails_fingerprint_proc
|
22
22
|
|
23
|
-
task :
|
23
|
+
task ci: ['engine_cart:generate'] do
|
24
24
|
# run the tests
|
25
25
|
Rake::Task['spec'].invoke
|
26
26
|
Rake::Task['doc'].invoke
|
@@ -35,15 +35,12 @@ begin
|
|
35
35
|
|
36
36
|
YARD::Rake::YardocTask.new(:doc) do |yt|
|
37
37
|
yt.files = Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
|
38
|
-
|
38
|
+
[File.join(project_root, 'README.rdoc')]
|
39
39
|
yt.options = ['--output-dir', doc_dest_dir, '--readme', 'README.rdoc', '--title', 'Discovery Indexer Documentation']
|
40
40
|
end
|
41
41
|
rescue LoadError
|
42
|
-
desc
|
42
|
+
desc 'Generate YARD Documentation'
|
43
43
|
task :doc do
|
44
|
-
abort
|
44
|
+
abort 'Please install the YARD gem to generate rdoc.'
|
45
45
|
end
|
46
|
-
end
|
47
|
-
|
48
|
-
|
49
|
-
|
46
|
+
end
|
@@ -1,24 +1,20 @@
|
|
1
1
|
module BaseIndexer
|
2
2
|
class AboutController < ApplicationController
|
3
|
-
|
4
3
|
def index
|
5
4
|
render text: 'ok', status: 200
|
6
5
|
end
|
7
|
-
|
6
|
+
|
8
7
|
def version
|
9
|
-
|
10
|
-
@result
|
11
|
-
@result.update(
|
12
|
-
|
13
|
-
|
8
|
+
@result = { app_name: Rails.configuration.app_name, rails_env: Rails.env, version: Rails.configuration.app_version, last_restart: (File.exist?('tmp/restart.txt') ? File.new('tmp/restart.txt').mtime : 'n/a'), last_deploy: (File.exist?('REVISION') ? File.new('REVISION').mtime : 'n/a'), revision: (File.exist?('REVISION') ? File.read('REVISION') : 'n/a') }
|
9
|
+
@result.update(solr_cores: BaseIndexer.solr_configuration_class_name.constantize.instance.get_configuration_hash)
|
10
|
+
@result.update(gems: Gem.loaded_specs) if request.format.html?
|
11
|
+
|
14
12
|
respond_to do |format|
|
15
|
-
format.json {render json: @result.to_json}
|
16
|
-
format.xml {render json: result.to_xml(root: 'status')}
|
17
|
-
format.html {render}
|
13
|
+
format.json { render json: @result.to_json }
|
14
|
+
format.xml { render json: result.to_xml(root: 'status') }
|
15
|
+
format.html { render }
|
18
16
|
# add the solr core names
|
19
|
-
end
|
20
|
-
|
17
|
+
end
|
21
18
|
end
|
22
|
-
|
23
19
|
end
|
24
|
-
end
|
20
|
+
end
|
@@ -1,19 +1,17 @@
|
|
1
1
|
module BaseIndexer
|
2
2
|
class ApplicationController < ActionController::Base
|
3
|
-
|
4
3
|
respond_to :json, :xml, :html
|
5
4
|
|
6
|
-
def remove_prefix
|
7
|
-
druid.gsub('druid:','') # lop off druid prefix if sent
|
5
|
+
def remove_prefix(druid)
|
6
|
+
druid.gsub('druid:', '') # lop off druid prefix if sent
|
8
7
|
end
|
9
|
-
|
10
|
-
def report_failure
|
11
|
-
|
8
|
+
|
9
|
+
def report_failure(method_symbol, params, e)
|
10
|
+
"#{method_symbol} #{params}\n\n#{e.inspect}\n#{e.message}\n#{e.backtrace}"
|
12
11
|
end
|
13
|
-
|
12
|
+
|
14
13
|
def report_success
|
15
|
-
|
14
|
+
'success'
|
16
15
|
end
|
17
|
-
|
18
16
|
end
|
19
17
|
end
|
@@ -1,47 +1,41 @@
|
|
1
|
-
require_dependency
|
1
|
+
require_dependency 'base_indexer/application_controller'
|
2
2
|
|
3
3
|
module BaseIndexer
|
4
4
|
class CollectionsController < ApplicationController
|
5
|
-
|
6
5
|
def new
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
6
|
+
druid = remove_prefix params[:druid]
|
7
|
+
|
8
|
+
Rails.logger.debug "Receiving indexing of collection #{druid}"
|
9
|
+
targets = params[:subtargets]
|
10
|
+
|
11
|
+
# initial collection item itself
|
12
|
+
indexer = BaseIndexer.indexer_class.constantize.new
|
13
|
+
indexer.index druid, targets
|
14
|
+
|
15
|
+
# initialize dor-fetcher to get list of druids for this collection
|
16
|
+
df = DorFetcher::Client.new(service_url: Rails.application.config.dor_fetcher_url)
|
17
|
+
|
18
|
+
item_druids = df.druid_array(df.get_collection(druid, {}))
|
16
19
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
item_druids.each do |druid|
|
27
|
-
|
28
|
-
druid=remove_prefix druid
|
29
|
-
counter+=1
|
30
|
-
indexer.index druid,targets
|
31
|
-
Rails.logger.debug "#{counter} of #{item_druids.size}: #{druid}"
|
32
|
-
|
33
|
-
end
|
34
|
-
|
35
|
-
@status = report_success
|
36
|
-
render nothing: true, status: 200
|
37
|
-
Rails.logger.debug "Completing indexing of collection #{druid}"
|
38
|
-
|
39
|
-
rescue Exception => e
|
40
|
-
@status = report_failure request.method_symbol, params, e
|
41
|
-
Rails.logger.error @status
|
42
|
-
render nothing: true, status: 202
|
20
|
+
Rails.logger.debug "Found #{item_druids.size} members of the collection #{druid}"
|
21
|
+
|
22
|
+
counter = 0
|
23
|
+
|
24
|
+
item_druids.each do |druid|
|
25
|
+
druid = remove_prefix druid
|
26
|
+
counter += 1
|
27
|
+
indexer.index druid, targets
|
28
|
+
Rails.logger.debug "#{counter} of #{item_druids.size}: #{druid}"
|
43
29
|
end
|
44
|
-
end
|
45
30
|
|
31
|
+
@status = report_success
|
32
|
+
render nothing: true, status: 200
|
33
|
+
Rails.logger.debug "Completing indexing of collection #{druid}"
|
34
|
+
|
35
|
+
rescue Exception => e
|
36
|
+
@status = report_failure request.method_symbol, params, e
|
37
|
+
Rails.logger.error @status
|
38
|
+
render nothing: true, status: 202
|
39
|
+
end
|
46
40
|
end
|
47
41
|
end
|
@@ -1,42 +1,36 @@
|
|
1
|
-
require_dependency
|
1
|
+
require_dependency 'base_indexer/application_controller'
|
2
2
|
|
3
3
|
module BaseIndexer
|
4
4
|
class ItemsController < ApplicationController
|
5
|
-
|
6
5
|
def new
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
render nothing: true, status: 202
|
21
|
-
end
|
6
|
+
druid = remove_prefix params[:druid]
|
7
|
+
Rails.logger.debug "Receiving indexing #{druid}"
|
8
|
+
targets = params[:subtargets]
|
9
|
+
|
10
|
+
indexer = BaseIndexer.indexer_class.constantize.new
|
11
|
+
indexer.index druid, targets
|
12
|
+
@status = report_success
|
13
|
+
render nothing: true, status: 200
|
14
|
+
Rails.logger.debug "Completing indexing #{druid}"
|
15
|
+
rescue StandardError => e
|
16
|
+
@status = report_failure request.method_symbol, params, e
|
17
|
+
Rails.logger.error @status
|
18
|
+
render nothing: true, status: 202
|
22
19
|
end
|
23
|
-
|
20
|
+
|
24
21
|
def destroy
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
render nothing: true, status: 202
|
38
|
-
end
|
22
|
+
druid = remove_prefix params[:druid]
|
23
|
+
Rails.logger.debug "Receiving deleting #{druid}"
|
24
|
+
|
25
|
+
indexer = BaseIndexer.indexer_class.constantize.new
|
26
|
+
indexer.delete druid
|
27
|
+
@status = report_success
|
28
|
+
render nothing: true, status: 200
|
29
|
+
Rails.logger.debug "Completing deleting #{druid}"
|
30
|
+
rescue StandardError => e
|
31
|
+
@status = report_failure request.method_symbol, params, e
|
32
|
+
Rails.logger.error @status
|
33
|
+
render nothing: true, status: 202
|
39
34
|
end
|
40
|
-
|
41
35
|
end
|
42
36
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
|
2
|
-
# Define the indexer class that will be used by the engine.
|
2
|
+
# Define the indexer class that will be used by the engine.
|
3
3
|
# The engine consumer app should override this class
|
4
|
-
BaseIndexer.indexer_class =
|
5
|
-
BaseIndexer.solr_configuration_class_name =
|
6
|
-
#BaseIndexer.solr_configuration_class.constantize.new(Rails.configuration.solr_config_file_path)
|
7
|
-
BaseIndexer.mapper_class_name =
|
8
|
-
BaseIndexer.solr_writer_class_name =
|
4
|
+
BaseIndexer.indexer_class = 'BaseIndexer::MainIndexerEngine'
|
5
|
+
BaseIndexer.solr_configuration_class_name = 'BaseIndexer::SolrConfigurationFromFile'
|
6
|
+
# BaseIndexer.solr_configuration_class.constantize.new(Rails.configuration.solr_config_file_path)
|
7
|
+
BaseIndexer.mapper_class_name = 'DiscoveryIndexer::Mapper::GeneralMapper'
|
8
|
+
BaseIndexer.solr_writer_class_name = 'DiscoveryIndexer::Writer::SolrWriter'
|
data/config/routes.rb
CHANGED
data/lib/base_indexer.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require
|
1
|
+
require 'base_indexer/engine'
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require 'base_indexer/main_indexer_engine'
|
4
|
+
require 'base_indexer/solr/solr_configuration'
|
5
|
+
require 'base_indexer/solr/solr_configuration_from_file'
|
6
|
+
require 'base_indexer/collection'
|
7
7
|
require 'discovery-indexer'
|
8
8
|
module BaseIndexer
|
9
9
|
mattr_accessor :indexer_class
|
@@ -11,4 +11,3 @@ module BaseIndexer
|
|
11
11
|
mattr_accessor :solr_writer_class_name
|
12
12
|
mattr_accessor :solr_configuration_class_name
|
13
13
|
end
|
14
|
-
|
data/lib/base_indexer/engine.rb
CHANGED
@@ -3,29 +3,27 @@ require 'active_support/core_ext/numeric/bytes'
|
|
3
3
|
module BaseIndexer
|
4
4
|
class Engine < ::Rails::Engine
|
5
5
|
isolate_namespace BaseIndexer
|
6
|
-
|
6
|
+
|
7
7
|
# Initialize memory store cache with 50 MB size
|
8
|
-
config.cache_store = [:memory_store, {:
|
9
|
-
|
8
|
+
config.cache_store = [:memory_store, { size: 64.megabytes }]
|
9
|
+
|
10
10
|
config.generators do |g|
|
11
11
|
g.test_framework :rspec
|
12
12
|
end
|
13
|
-
config.app_version =
|
14
|
-
config.app_name =
|
15
|
-
|
13
|
+
config.app_version = '0.0'
|
14
|
+
config.app_name = '[You have to override this name in your app]'
|
15
|
+
|
16
16
|
config.after_initialize do
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
if File.exists? config.solr_config_file
|
17
|
+
config.solr_config_file = File.join(Rails.root, 'config', 'solr.yml')
|
18
|
+
|
19
|
+
if File.exist? config.solr_config_file
|
21
20
|
# Reads the SOLR configuration file
|
22
21
|
BaseIndexer.solr_configuration_class_name.constantize.instance.read(config.solr_config_file)
|
23
22
|
end
|
24
|
-
|
25
|
-
# Initializes the DiscoveryIndexer log with Rails logger, so all the messages will go to
|
23
|
+
|
24
|
+
# Initializes the DiscoveryIndexer log with Rails logger, so all the messages will go to
|
26
25
|
# the same log file
|
27
26
|
DiscoveryIndexer::Logging.logger = Rails.logger
|
28
27
|
end
|
29
28
|
end
|
30
29
|
end
|
31
|
-
|
data/lib/base_indexer/version.rb
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
require 'rails/generators'
|
2
2
|
|
3
3
|
module BaseIndexer
|
4
|
-
|
4
|
+
class Install < Rails::Generators::Base
|
5
|
+
source_root File.expand_path('../templates', __FILE__)
|
5
6
|
|
6
|
-
|
7
|
-
|
8
|
-
desc "Install Base Indexer"
|
7
|
+
desc 'Install Base Indexer'
|
9
8
|
|
10
9
|
def assets
|
11
|
-
copy_file
|
10
|
+
copy_file 'solr.yml', 'config/solr.yml'
|
12
11
|
end
|
13
|
-
|
14
|
-
end
|
12
|
+
end
|
13
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: base_indexer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ahmed Alsum
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-10-
|
12
|
+
date: 2015-10-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -37,14 +37,14 @@ dependencies:
|
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.9.
|
40
|
+
version: 0.9.11
|
41
41
|
type: :runtime
|
42
42
|
prerelease: false
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.9.
|
47
|
+
version: 0.9.11
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: retries
|
50
50
|
requirement: !ruby/object:Gem::Requirement
|
@@ -215,7 +215,6 @@ dependencies:
|
|
215
215
|
version: '0'
|
216
216
|
description: Description of BaseIndexer.
|
217
217
|
email:
|
218
|
-
- aalsum@stanford.edu
|
219
218
|
- laneymcg@stanford.edu
|
220
219
|
executables: []
|
221
220
|
extensions: []
|