cdmdexer 0.20.0 → 0.22.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.env.example +2 -0
- data/.github/workflows/ruby.yml +36 -0
- data/.gitignore +4 -0
- data/README.md +12 -3
- data/cdmdexer.gemspec +14 -9
- data/lib/cdmdexer/default_solr.rb +12 -11
- data/lib/cdmdexer/etl_worker.rb +10 -1
- data/lib/cdmdexer/formatters.rb +1 -1
- data/lib/cdmdexer/oai_request.rb +22 -5
- data/lib/cdmdexer/transformer.rb +3 -114
- data/lib/cdmdexer/version.rb +2 -2
- data/lib/generators/cdmdexer/install_generator.rb +14 -0
- data/lib/generators/cdmdexer/settings_generator.rb +21 -0
- data/lib/generators/cdmdexer/templates/cdmdexer.yml +590 -0
- data/lib/generators/cdmdexer/templates/rails_config.rb +7 -0
- metadata +88 -26
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3c121edb1dcd83e2ec05efa244b9648f00d9868bfdf8645ea8f93a2d1e0cd4e4
|
|
4
|
+
data.tar.gz: 53bc284ed3449d418f9e5dee425dc1d8c15d585c726cb1670279bd839701622f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3df3d7fd00d6ec4e20cd12600548880b93a0cb75a0b3c5ec837f36e2b6075e06a7741796226d6df000cd4b6eb3180e18bd8952b77ff2293537748169d2168947
|
|
7
|
+
data.tar.gz: 9ef7b1c0a257697eb99f7a153d1dddf183a9fab5475c0bbd706ad011f5a0fd1c6ad7b993e0578a8f90a8107c370cdc70a1ac148ed69f271a0f659347057e3463
|
data/.env.example
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
2
|
+
# They are provided by a third-party and are governed by
|
|
3
|
+
# separate terms of service, privacy policy, and support
|
|
4
|
+
# documentation.
|
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
|
7
|
+
|
|
8
|
+
name: Ruby
|
|
9
|
+
|
|
10
|
+
on: [push]
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
test:
|
|
14
|
+
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
strategy:
|
|
17
|
+
matrix:
|
|
18
|
+
ruby-version: ['2.6', '2.7', '3.0']
|
|
19
|
+
|
|
20
|
+
steps:
|
|
21
|
+
- name: Checkout code repo
|
|
22
|
+
uses: actions/checkout@v2
|
|
23
|
+
|
|
24
|
+
- name: Set up Ruby
|
|
25
|
+
uses: ruby/setup-ruby@v1
|
|
26
|
+
with:
|
|
27
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
28
|
+
|
|
29
|
+
- name: Run bundler
|
|
30
|
+
run: bundle install
|
|
31
|
+
|
|
32
|
+
- name: Run tests
|
|
33
|
+
env:
|
|
34
|
+
GEONAMES_USER: ${{ secrets.GEONAMES_USER }}
|
|
35
|
+
GEONAMES_TOKEN: ${{ secrets.GEONAMES_TOKEN }}
|
|
36
|
+
run: bundle exec rake test
|
data/.gitignore
CHANGED
data/README.md
CHANGED
|
@@ -20,6 +20,10 @@ Or install it yourself as:
|
|
|
20
20
|
|
|
21
21
|
$ gem install cdmdexer
|
|
22
22
|
|
|
23
|
+
Run the cdmdexer install generator. This step will add a `config/settings.yml` file to your local application, containing default Solr field mappings and formatters to your project:
|
|
24
|
+
|
|
25
|
+
$ bundle exec rails g cdmdexer:install
|
|
26
|
+
|
|
23
27
|
Add the CDMDEXER rake task to your project Rakefile:
|
|
24
28
|
|
|
25
29
|
```ruby
|
|
@@ -30,9 +34,14 @@ require 'cdmdexer/rake_task'
|
|
|
30
34
|
|
|
31
35
|
In order to make use of the GeoNames service, you must purchase a [GeoNames Premium Webservices Account](http://www.geonames.org/commercial-webservices.html). If you do not have a `geonam` field in your CONTENTdm schema, you may ignore this instruction. Add your credentials to your shell environment once you have secured a GeoNames user:
|
|
32
36
|
|
|
37
|
+
|
|
33
38
|
```
|
|
34
|
-
|
|
35
|
-
|
|
39
|
+
cp .env.example .env
|
|
40
|
+
nano .env
|
|
41
|
+
|
|
42
|
+
# Add these vars to the .env file
|
|
43
|
+
GEONAMES_USER=foo
|
|
44
|
+
GEONAMES_TOKEN=bar
|
|
36
45
|
```
|
|
37
46
|
|
|
38
47
|
## Usage
|
|
@@ -178,4 +187,4 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/UMNLib
|
|
|
178
187
|
## TODO
|
|
179
188
|
|
|
180
189
|
* Make StripFormatter the default formatter so it doesn't need to be declared for every field
|
|
181
|
-
* Re-brand project: CONTENTdm Indexer. CDMDEXER doesn't necessarily require Blacklight. Moreover only handles indexing.
|
|
190
|
+
* Re-brand project: CONTENTdm Indexer. CDMDEXER doesn't necessarily require Blacklight. Moreover only handles indexing.
|
data/cdmdexer.gemspec
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
require 'cdmdexer/version'
|
|
5
6
|
|
|
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
|
9
10
|
spec.authors = ['chadfennell']
|
|
10
11
|
spec.email = ['fenne035@umn.edu']
|
|
11
12
|
|
|
12
|
-
spec.summary =
|
|
13
|
+
spec.summary = 'Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails application.'
|
|
13
14
|
spec.license = 'MIT'
|
|
14
15
|
|
|
15
16
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
@@ -17,19 +18,23 @@ Gem::Specification.new do |spec|
|
|
|
17
18
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
18
19
|
spec.require_paths = ['lib']
|
|
19
20
|
|
|
20
|
-
spec.add_dependency '
|
|
21
|
-
spec.add_dependency 'contentdm_api', '~> 0.
|
|
21
|
+
spec.add_dependency 'config', '~> 3.1'
|
|
22
|
+
spec.add_dependency 'contentdm_api', '~> 0.6.0'
|
|
23
|
+
spec.add_dependency 'hash_at_path', '~> 0.1.6'
|
|
24
|
+
spec.add_dependency 'rsolr', '~> 2.0'
|
|
22
25
|
spec.add_dependency 'sidekiq', '>= 3.5'
|
|
23
26
|
spec.add_dependency 'titleize', '~> 1.4'
|
|
24
|
-
spec.add_dependency 'rsolr', '~> 2.0'
|
|
25
27
|
# CDMDEXER expects to run in a rails app, but just to avoid adding
|
|
26
28
|
# another external dependency for XML procssing, we rely on activesupport's
|
|
27
29
|
# Has.to_jsonl feature for testing and to allow this gem to function
|
|
28
30
|
# independently from a rails app
|
|
29
|
-
spec.add_dependency 'rails', '
|
|
31
|
+
spec.add_dependency 'rails', '~> 6.0'
|
|
32
|
+
spec.add_dependency 'rexml', '~> 3.2'
|
|
30
33
|
|
|
31
|
-
spec.add_development_dependency '
|
|
32
|
-
spec.add_development_dependency '
|
|
34
|
+
spec.add_development_dependency 'dotenv-rails', '~> 2.7.6'
|
|
35
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
|
36
|
+
spec.add_development_dependency 'byebug', '~> 11.1'
|
|
33
37
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
|
38
|
+
spec.add_development_dependency 'rake', '~> 12.0'
|
|
34
39
|
spec.add_development_dependency 'yard', '~> 0.9.0'
|
|
35
40
|
end
|
|
@@ -1,27 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'rsolr'
|
|
2
4
|
|
|
3
5
|
module CDMDEXER
|
|
4
6
|
# Commnicate with Solr: add / delete stuff
|
|
5
7
|
class DefaultSolr
|
|
6
8
|
attr_reader :url, :client
|
|
7
|
-
|
|
8
|
-
|
|
9
|
+
|
|
10
|
+
def initialize(url = 'http://localhost:8983/solr/blacklight-core', client = RSolr)
|
|
11
|
+
@url = ENV['SOLR_URL'] || url
|
|
9
12
|
@client = client
|
|
10
13
|
end
|
|
11
14
|
|
|
12
15
|
def ids(start: 0)
|
|
13
16
|
connection.get('select',
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
}
|
|
20
|
-
)
|
|
17
|
+
params: { q: '*:*',
|
|
18
|
+
defType: 'edismax',
|
|
19
|
+
fl: '',
|
|
20
|
+
rows: 10,
|
|
21
|
+
start: start })
|
|
21
22
|
end
|
|
22
23
|
|
|
23
24
|
def connection
|
|
24
|
-
@connection ||= client.connect url: url
|
|
25
|
+
@connection ||= client.connect url: @url
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
def add(records)
|
|
@@ -32,4 +33,4 @@ module CDMDEXER
|
|
|
32
33
|
connection.delete_by_id ids
|
|
33
34
|
end
|
|
34
35
|
end
|
|
35
|
-
end
|
|
36
|
+
end
|
data/lib/cdmdexer/etl_worker.rb
CHANGED
|
@@ -38,11 +38,20 @@ module CDMDEXER
|
|
|
38
38
|
@resumption_token = config.fetch('resumption_token', nil)
|
|
39
39
|
@batch_size = config.fetch('batch_size', 5).to_i
|
|
40
40
|
@is_recursive = config.fetch('is_recursive', true)
|
|
41
|
+
after_date = config.fetch('after_date', false)
|
|
41
42
|
|
|
42
43
|
@oai_request = oai_request_klass.new(
|
|
43
44
|
endpoint_url: oai_endpoint,
|
|
44
45
|
resumption_token: resumption_token,
|
|
45
|
-
set_spec: config.fetch('set_spec', nil)
|
|
46
|
+
set_spec: config.fetch('set_spec', nil),
|
|
47
|
+
# Optionally only select records that have been updated after a
|
|
48
|
+
# certain date. You may need to manually update a parent record
|
|
49
|
+
# after updating a child in order to signify to the indexer that
|
|
50
|
+
# some record in the parent's children has been updated. This indexer
|
|
51
|
+
# expects to only see parent records in the OAI responses.
|
|
52
|
+
# The default here is to skip indexing based on date.
|
|
53
|
+
# Rails example for getting a date: `after_date: 2.weeks.ago`
|
|
54
|
+
after_date: after_date
|
|
46
55
|
)
|
|
47
56
|
|
|
48
57
|
run_batch!
|
data/lib/cdmdexer/formatters.rb
CHANGED
data/lib/cdmdexer/oai_request.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'hash_at_path'
|
|
2
4
|
require 'json'
|
|
5
|
+
require 'time'
|
|
3
6
|
|
|
4
7
|
module CDMDEXER
|
|
5
8
|
# Light wrapper around OAI requests
|
|
@@ -10,16 +13,19 @@ module CDMDEXER
|
|
|
10
13
|
attr_reader :endpoint_url,
|
|
11
14
|
:resumption_token,
|
|
12
15
|
:client,
|
|
13
|
-
:set_spec
|
|
16
|
+
:set_spec,
|
|
17
|
+
:after_date
|
|
14
18
|
|
|
15
19
|
def initialize(endpoint_url: '',
|
|
16
20
|
resumption_token: nil,
|
|
17
21
|
set_spec: nil,
|
|
18
|
-
client: Net::HTTP
|
|
22
|
+
client: Net::HTTP,
|
|
23
|
+
after_date: false)
|
|
19
24
|
@endpoint_url = endpoint_url
|
|
20
25
|
@resumption_token = resumption_token
|
|
21
26
|
@client = client
|
|
22
27
|
@set_spec = set_spec ? "&set=#{set_spec}" : ''
|
|
28
|
+
@after_date = after_date
|
|
23
29
|
end
|
|
24
30
|
|
|
25
31
|
def records
|
|
@@ -44,12 +50,23 @@ module CDMDEXER
|
|
|
44
50
|
end
|
|
45
51
|
|
|
46
52
|
def deletable_ids
|
|
47
|
-
records.select
|
|
48
|
-
|
|
53
|
+
records.select do |record|
|
|
54
|
+
if record['status'] == 'deleted'
|
|
55
|
+
after_date ? Time.parse(record['datestamp']) >= after_date : true
|
|
56
|
+
end
|
|
57
|
+
end.map { |record| record[:id] }
|
|
49
58
|
end
|
|
50
59
|
|
|
51
60
|
def updatables
|
|
52
|
-
records.reject
|
|
61
|
+
records.reject do |record|
|
|
62
|
+
if record['status'] == 'deleted'
|
|
63
|
+
true
|
|
64
|
+
elsif after_date && Time.parse(record['datestamp']) < after_date
|
|
65
|
+
true
|
|
66
|
+
else
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
53
70
|
end
|
|
54
71
|
|
|
55
72
|
private
|
data/lib/cdmdexer/transformer.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'json'
|
|
2
2
|
require 'titleize'
|
|
3
|
+
require 'active_support/core_ext/integer/time'
|
|
3
4
|
|
|
4
5
|
module CDMDEXER
|
|
5
6
|
class Transformer
|
|
@@ -53,119 +54,7 @@ module CDMDEXER
|
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
def default_field_mappings
|
|
56
|
-
|
|
57
|
-
{dest_path: 'location_llsi', origin_path: '/', formatters: [LocationFormatter]},
|
|
58
|
-
{dest_path: 'id', origin_path: 'id', formatters: [StripFormatter]},
|
|
59
|
-
{dest_path: 'setspec_ssi', origin_path: '/', formatters: [AddSetSpecFormatter, SetSpecFormatter]},
|
|
60
|
-
{dest_path: 'collection_name_ssi', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionNameFormatter]},
|
|
61
|
-
{dest_path: 'collection_name_tei', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionNameFormatter]},
|
|
62
|
-
{dest_path: 'collection_description_tesi', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionDescriptionFormatter, FilterBadCollections]},
|
|
63
|
-
{dest_path: 'parent_collection_name_ssi', origin_path: 'par', formatters: [StripFormatter]},
|
|
64
|
-
{dest_path: 'parent_collection_name_tei', origin_path: 'par', formatters: [StripFormatter]},
|
|
65
|
-
{dest_path: 'parent_collection_description_tei', origin_path: 'par', formatters: [StripFormatter]},
|
|
66
|
-
{dest_path: 'title_tesi', origin_path: 'title', formatters: [StripFormatter]},
|
|
67
|
-
{dest_path: 'title_ssi', origin_path: 'title', formatters: [StripFormatter]},
|
|
68
|
-
{dest_path: 'title_sort', origin_path: 'title', formatters: [StripFormatter]},
|
|
69
|
-
{dest_path: 'title_unstem_search', origin_path: 'title', formatters: [StripFormatter]},
|
|
70
|
-
{dest_path: 'contributor_teim', origin_path: 'contri', formatters: [StripFormatter]},
|
|
71
|
-
{dest_path: 'contributor_unstem_search', origin_path: 'contri', formatters: [StripFormatter]},
|
|
72
|
-
{dest_path: 'contributor_ssim', origin_path: 'contri', formatters: [SplitFormatter, StripFormatter]},
|
|
73
|
-
{dest_path: 'creator_tesi', origin_path: 'photog', formatters: [JoinFormatter, StripFormatter]},
|
|
74
|
-
{dest_path: 'creator_unstem_search', origin_path: 'photog', formatters: [StripFormatter]},
|
|
75
|
-
{dest_path: 'creator_ssim', origin_path: 'photog', formatters: [SplitFormatter, StripFormatter]},
|
|
76
|
-
{dest_path: 'creator_sort', origin_path: 'photog', formatters: [StripFormatter]},
|
|
77
|
-
{dest_path: 'description_tei', origin_path: 'descri', formatters: [StripFormatter]},
|
|
78
|
-
{dest_path: 'description_ts', origin_path: 'descri', formatters: [StripFormatter]},
|
|
79
|
-
{dest_path: 'dat_ssi', origin_path: 'dat', formatters: [StripFormatter]},
|
|
80
|
-
{dest_path: 'dat_tesi', origin_path: 'dat', formatters: [StripFormatter]},
|
|
81
|
-
{dest_path: 'dat_sort', origin_path: 'dat', formatters: [StripFormatter]},
|
|
82
|
-
{dest_path: 'publishing_agency_tei', origin_path: 'publia', formatters: [StripFormatter]},
|
|
83
|
-
{dest_path: 'publishing_agency_unstem_search', origin_path: 'publia', formatters: [StripFormatter]},
|
|
84
|
-
{dest_path: 'publishing_agency_ssi', origin_path: 'publia', formatters: [StripFormatter]},
|
|
85
|
-
{dest_path: 'dimensions_ssi', origin_path: 'dimens', formatters: [StripFormatter]},
|
|
86
|
-
{dest_path: 'topic_teim', origin_path: 'genera', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
|
87
|
-
{dest_path: 'topic_ssim', origin_path: 'genera', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
|
88
|
-
{dest_path: 'topic_unstem_search', origin_path: 'genera', formatters: [StripSemicolonFormatter, StripFormatter]},
|
|
89
|
-
{dest_path: 'type_ssi', origin_path: 'type', formatters: [Titlieze, StripSemicolonFormatter, StripFormatter]},
|
|
90
|
-
{dest_path: 'type_tesi', origin_path: 'type', formatters: [Titlieze, StripSemicolonFormatter, StripFormatter]},
|
|
91
|
-
{dest_path: 'physical_format_ssi', origin_path: 'physic', formatters: [StripSemicolonFormatter]},
|
|
92
|
-
{dest_path: 'physical_format_tesi', origin_path: 'physic', formatters: [StripSemicolonFormatter]},
|
|
93
|
-
{dest_path: 'formal_subject_unstem_search', origin_path: 'specif', formatters: [StripFormatter]},
|
|
94
|
-
{dest_path: 'formal_subject_ssim', origin_path: 'specif', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
|
95
|
-
{dest_path: 'formal_subject_teim', origin_path: 'specif', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
|
96
|
-
{dest_path: 'subject_unstem_search', origin_path: 'subjec', formatters: [StripFormatter]},
|
|
97
|
-
{dest_path: 'subject_teim', origin_path: 'subjec', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
|
98
|
-
{dest_path: 'subject_ssim', origin_path: 'subjec', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
|
99
|
-
{dest_path: 'keyword_unstem_search', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, StripFormatter]},
|
|
100
|
-
{dest_path: 'keyword_tesi', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, JoinFormatter, StripFormatter]},
|
|
101
|
-
{dest_path: 'keyword_ssim', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, StripFormatter]},
|
|
102
|
-
{dest_path: 'city_ssim', origin_path: 'city', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
|
103
|
-
{dest_path: 'city_unstem_search', origin_path: 'city', formatters: [StripFormatter]},
|
|
104
|
-
{dest_path: 'district_ssi', origin_path: 'distri', formatters: [StripFormatter]},
|
|
105
|
-
{dest_path: 'district_unstem_search', origin_path: 'distri', formatters: [StripFormatter]},
|
|
106
|
-
{dest_path: 'county_ssim', origin_path: 'county', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
|
107
|
-
{dest_path: 'county_unstem_search', origin_path: 'county', formatters: [StripFormatter]},
|
|
108
|
-
{dest_path: 'state_ssi', origin_path: 'state', formatters: [StripFormatter]},
|
|
109
|
-
{dest_path: 'state_unstem_search', origin_path: 'state', formatters: [StripFormatter]},
|
|
110
|
-
{dest_path: 'country_ssi', origin_path: 'countr', formatters: [StripFormatter]},
|
|
111
|
-
{dest_path: 'country_unstem_search', origin_path: 'countr', formatters: [StripFormatter]},
|
|
112
|
-
{dest_path: 'language_ssi', origin_path: 'langua', formatters: [StripFormatter]},
|
|
113
|
-
{dest_path: 'language_unstem_search', origin_path: 'langua', formatters: [StripFormatter]},
|
|
114
|
-
{dest_path: 'contributing_unstem_search', origin_path: 'contra', formatters: [StripFormatter]},
|
|
115
|
-
{dest_path: 'contributing_organization_tesi', origin_path: 'contra', formatters: [StripFormatter]},
|
|
116
|
-
{dest_path: 'contributing_organization_ssi', origin_path: 'contra', formatters: [Titlieze, StripFormatter]},
|
|
117
|
-
{dest_path: 'contact_information_ssi', origin_path: 'contac', formatters: [StripFormatter]},
|
|
118
|
-
{dest_path: 'rights_ssi', origin_path: 'righta', formatters: [StripFormatter]},
|
|
119
|
-
{dest_path: 'local_identifier_ssi', origin_path: 'identi', formatters: [StripFormatter]},
|
|
120
|
-
{dest_path: 'identifier_ssi', origin_path: 'resour', formatters: [StripFormatter]},
|
|
121
|
-
{dest_path: 'project_ssi', origin_path: 'projec', formatters: [StripFormatter]},
|
|
122
|
-
{dest_path: 'fiscal_sponsor_ssi', origin_path: 'fiscal', formatters: [StripFormatter]},
|
|
123
|
-
{dest_path: 'publisher_ssi', origin_path: 'publis', formatters: [StripFormatter]},
|
|
124
|
-
{dest_path: 'date_ssi', origin_path: 'date', formatters: [StripFormatter]},
|
|
125
|
-
{dest_path: 'format_tesi', origin_path: 'format', formatters: [StripFormatter]},
|
|
126
|
-
{dest_path: 'digspa_ssi', origin_path: 'digspa'},
|
|
127
|
-
{dest_path: 'digspb_ssi', origin_path: 'digspb'},
|
|
128
|
-
{dest_path: 'digspc_ssi', origin_path: 'digspc'},
|
|
129
|
-
{dest_path: 'digspd_ssi', origin_path: 'digspd'},
|
|
130
|
-
{dest_path: 'digspe_ssi', origin_path: 'digspe'},
|
|
131
|
-
{dest_path: 'digspf_ssi', origin_path: 'digspf'},
|
|
132
|
-
{dest_path: 'digspg_ssi', origin_path: 'digspg'},
|
|
133
|
-
{dest_path: 'digsph_ssi', origin_path: 'digsph'},
|
|
134
|
-
{dest_path: 'digspi_ssi', origin_path: 'digspi'},
|
|
135
|
-
{dest_path: 'digspj_ssi', origin_path: 'digspj'},
|
|
136
|
-
{dest_path: 'digspk_ssi', origin_path: 'digspk'},
|
|
137
|
-
{dest_path: 'transcription_tesi', origin_path: 'transc', formatters: [StripFormatter]},
|
|
138
|
-
{dest_path: 'translation_tesi', origin_path: 'transl', formatters: [StripFormatter]},
|
|
139
|
-
{dest_path: 'fullrs_tesi', origin_path: 'fullrs', formatters: [StripFormatter]},
|
|
140
|
-
{dest_path: 'find_ssi', origin_path: 'find', formatters: [StripFormatter]},
|
|
141
|
-
{dest_path: 'dmaccess_ssi', origin_path: 'dmaccess', formatters: [StripFormatter]},
|
|
142
|
-
{dest_path: 'dmimage_ssi', origin_path: 'dmimage', formatters: [StripFormatter]},
|
|
143
|
-
{dest_path: 'dmcreated_ssi', origin_path: 'dmcreated', formatters: [StripFormatter]},
|
|
144
|
-
{dest_path: 'dmmodified_ssi', origin_path: 'dmmodified', formatters: [StripFormatter]},
|
|
145
|
-
{dest_path: 'dmoclcno_ssi', origin_path: 'dmoclcno', formatters: [StripFormatter]},
|
|
146
|
-
{dest_path: 'restriction_code_ssi', origin_path: 'restrictionCode', formatters: [StripFormatter]},
|
|
147
|
-
{dest_path: 'cdmfilesize_ssi', origin_path: 'cdmfilesize', formatters: [StripFormatter]},
|
|
148
|
-
{dest_path: 'cdmfilesizeformatted_ssi', origin_path: 'cdmfilesizeformatted', formatters: [StripFormatter]},
|
|
149
|
-
{dest_path: 'cdmprintpdf_is', origin_path: 'cdmprintpdf', formatters: [ToIFormatter]},
|
|
150
|
-
{dest_path: 'cdmhasocr_is', origin_path: 'cdmhasocr', formatters: [ToIFormatter]},
|
|
151
|
-
{dest_path: 'cdmisnewspaper_is', origin_path: 'cdmisnewspaper', formatters: [ToIFormatter]},
|
|
152
|
-
{dest_path: 'image_uri_ssi', origin_path: 'image_uri', formatters: [StripFormatter]},
|
|
153
|
-
{dest_path: 'record_type_ssi', origin_path: 'record_type', formatters: [StripFormatter]},
|
|
154
|
-
{dest_path: 'geographic_feature_ssim', origin_path: 'geogra', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
|
155
|
-
{dest_path: 'geographic_feature_teim', origin_path: 'geogra', formatters: [StripFormatter]},
|
|
156
|
-
{dest_path: 'geographic_feature_unstem_search', origin_path: 'geogra', formatters: [StripFormatter]},
|
|
157
|
-
{dest_path: 'geonam_ssi', origin_path: 'geonam', formatters: [StripFormatter]},
|
|
158
|
-
{dest_path: 'kaltura_audio_ssi', origin_path: 'audio', formatters: [StripFormatter]},
|
|
159
|
-
{dest_path: 'kaltura_audio_playlist_ssi', origin_path: 'audioa', formatters: [StripFormatter]},
|
|
160
|
-
{dest_path: 'kaltura_video_ssi', origin_path: 'video', formatters: [StripFormatter]},
|
|
161
|
-
{dest_path: 'kaltura_video_playlist_ssi', origin_path: 'videoa', formatters: [StripFormatter]},
|
|
162
|
-
{dest_path: 'coordinates_llsi', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToLocation]},
|
|
163
|
-
{dest_path: 'placename_ssim', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToPlaceName]},
|
|
164
|
-
{dest_path: 'placename_unstem_search', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToPlaceName]},
|
|
165
|
-
{dest_path: 'table_ssim', origin_path: 'table', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
|
166
|
-
{dest_path: 'umedia_ssi', origin_path: 'umedia', formatters: [StripFormatter]},
|
|
167
|
-
{dest_path: 'child_index', origin_path: 'child_index', formatters: []}
|
|
168
|
-
]
|
|
57
|
+
Settings.field_mappings
|
|
169
58
|
end
|
|
170
59
|
end
|
|
171
|
-
end
|
|
60
|
+
end
|
data/lib/cdmdexer/version.rb
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
module CDMDEXER
|
|
2
|
-
VERSION = "0.
|
|
3
|
-
end
|
|
2
|
+
VERSION = "0.22.0"
|
|
3
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'rails/generators'
|
|
3
|
+
|
|
4
|
+
module Cdmdexer
|
|
5
|
+
class InstallGenerator < Rails::Generators::Base
|
|
6
|
+
source_root File.expand_path('../templates', __FILE__)
|
|
7
|
+
|
|
8
|
+
desc 'Install CDMDEXER'
|
|
9
|
+
|
|
10
|
+
def add_rails_config_settings
|
|
11
|
+
generate 'cdmdexer:settings'
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'rails/generators'
|
|
3
|
+
|
|
4
|
+
module Cdmdexer
|
|
5
|
+
class SettingsGenerator < Rails::Generators::Base
|
|
6
|
+
source_root ::File.expand_path('../templates', __FILE__)
|
|
7
|
+
|
|
8
|
+
desc <<-EOF
|
|
9
|
+
This generator makes the following changes to your application:
|
|
10
|
+
1. Installs settings.yml into your application's config directory
|
|
11
|
+
EOF
|
|
12
|
+
|
|
13
|
+
def add_initializer
|
|
14
|
+
copy_file 'rails_config.rb', 'config/initializers/rails_config.rb'
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def install_settings
|
|
18
|
+
copy_file 'cdmdexer.yml', 'config/initializers/cdmdexer.yml'
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
### Default Gem Field mappings
|
|
2
|
+
---
|
|
3
|
+
field_mappings:
|
|
4
|
+
-
|
|
5
|
+
dest_path: location_llsi
|
|
6
|
+
formatters:
|
|
7
|
+
- CDMDEXER::LocationFormatter
|
|
8
|
+
origin_path: /
|
|
9
|
+
-
|
|
10
|
+
dest_path: id
|
|
11
|
+
formatters:
|
|
12
|
+
- CDMDEXER::StripFormatter
|
|
13
|
+
origin_path: id
|
|
14
|
+
-
|
|
15
|
+
dest_path: setspec_ssi
|
|
16
|
+
formatters:
|
|
17
|
+
- CDMDEXER::AddSetSpecFormatter
|
|
18
|
+
- CDMDEXER::SetSpecFormatter
|
|
19
|
+
origin_path: /
|
|
20
|
+
-
|
|
21
|
+
dest_path: collection_name_ssi
|
|
22
|
+
formatters:
|
|
23
|
+
- CDMDEXER::AddSetSpecFormatter
|
|
24
|
+
- CDMDEXER::CollectionNameFormatter
|
|
25
|
+
origin_path: /
|
|
26
|
+
-
|
|
27
|
+
dest_path: collection_name_tei
|
|
28
|
+
formatters:
|
|
29
|
+
- CDMDEXER::AddSetSpecFormatter
|
|
30
|
+
- CDMDEXER::CollectionNameFormatter
|
|
31
|
+
origin_path: /
|
|
32
|
+
-
|
|
33
|
+
dest_path: collection_description_tesi
|
|
34
|
+
formatters:
|
|
35
|
+
- CDMDEXER::AddSetSpecFormatter
|
|
36
|
+
- CDMDEXER::CollectionDescriptionFormatter
|
|
37
|
+
- CDMDEXER::FilterBadCollections
|
|
38
|
+
origin_path: /
|
|
39
|
+
-
|
|
40
|
+
dest_path: parent_collection_name_ssi
|
|
41
|
+
formatters:
|
|
42
|
+
- CDMDEXER::StripFormatter
|
|
43
|
+
origin_path: par
|
|
44
|
+
-
|
|
45
|
+
dest_path: parent_collection_name_tei
|
|
46
|
+
formatters:
|
|
47
|
+
- CDMDEXER::StripFormatter
|
|
48
|
+
origin_path: par
|
|
49
|
+
-
|
|
50
|
+
dest_path: parent_collection_description_tei
|
|
51
|
+
formatters:
|
|
52
|
+
- CDMDEXER::StripFormatter
|
|
53
|
+
origin_path: par
|
|
54
|
+
-
|
|
55
|
+
dest_path: title_tesi
|
|
56
|
+
formatters:
|
|
57
|
+
- CDMDEXER::StripFormatter
|
|
58
|
+
origin_path: title
|
|
59
|
+
-
|
|
60
|
+
dest_path: title_ssi
|
|
61
|
+
formatters:
|
|
62
|
+
- CDMDEXER::StripFormatter
|
|
63
|
+
origin_path: title
|
|
64
|
+
-
|
|
65
|
+
dest_path: title_sort
|
|
66
|
+
formatters:
|
|
67
|
+
- CDMDEXER::StripFormatter
|
|
68
|
+
origin_path: title
|
|
69
|
+
-
|
|
70
|
+
dest_path: title_unstem_search
|
|
71
|
+
formatters:
|
|
72
|
+
- CDMDEXER::StripFormatter
|
|
73
|
+
origin_path: title
|
|
74
|
+
-
|
|
75
|
+
dest_path: contributor_teim
|
|
76
|
+
formatters:
|
|
77
|
+
- CDMDEXER::StripFormatter
|
|
78
|
+
origin_path: contri
|
|
79
|
+
-
|
|
80
|
+
dest_path: contributor_unstem_search
|
|
81
|
+
formatters:
|
|
82
|
+
- CDMDEXER::StripFormatter
|
|
83
|
+
origin_path: contri
|
|
84
|
+
-
|
|
85
|
+
dest_path: contributor_ssim
|
|
86
|
+
formatters:
|
|
87
|
+
- CDMDEXER::SplitFormatter
|
|
88
|
+
- CDMDEXER::StripFormatter
|
|
89
|
+
origin_path: contri
|
|
90
|
+
-
|
|
91
|
+
dest_path: creator_tesi
|
|
92
|
+
formatters:
|
|
93
|
+
- CDMDEXER::JoinFormatter
|
|
94
|
+
- CDMDEXER::StripFormatter
|
|
95
|
+
origin_path: photog
|
|
96
|
+
-
|
|
97
|
+
dest_path: creator_unstem_search
|
|
98
|
+
formatters:
|
|
99
|
+
- CDMDEXER::StripFormatter
|
|
100
|
+
origin_path: photog
|
|
101
|
+
-
|
|
102
|
+
dest_path: creator_ssim
|
|
103
|
+
formatters:
|
|
104
|
+
- CDMDEXER::SplitFormatter
|
|
105
|
+
- CDMDEXER::StripFormatter
|
|
106
|
+
origin_path: photog
|
|
107
|
+
-
|
|
108
|
+
dest_path: creator_sort
|
|
109
|
+
formatters:
|
|
110
|
+
- CDMDEXER::StripFormatter
|
|
111
|
+
origin_path: photog
|
|
112
|
+
-
|
|
113
|
+
dest_path: description_tei
|
|
114
|
+
formatters:
|
|
115
|
+
- CDMDEXER::StripFormatter
|
|
116
|
+
origin_path: descri
|
|
117
|
+
-
|
|
118
|
+
dest_path: description_ts
|
|
119
|
+
formatters:
|
|
120
|
+
- CDMDEXER::StripFormatter
|
|
121
|
+
origin_path: descri
|
|
122
|
+
-
|
|
123
|
+
dest_path: dat_ssi
|
|
124
|
+
formatters:
|
|
125
|
+
- CDMDEXER::StripFormatter
|
|
126
|
+
origin_path: dat
|
|
127
|
+
-
|
|
128
|
+
dest_path: dat_tesi
|
|
129
|
+
formatters:
|
|
130
|
+
- CDMDEXER::StripFormatter
|
|
131
|
+
origin_path: dat
|
|
132
|
+
-
|
|
133
|
+
dest_path: dat_sort
|
|
134
|
+
formatters:
|
|
135
|
+
- CDMDEXER::StripFormatter
|
|
136
|
+
origin_path: dat
|
|
137
|
+
-
|
|
138
|
+
dest_path: publishing_agency_tei
|
|
139
|
+
formatters:
|
|
140
|
+
- CDMDEXER::StripFormatter
|
|
141
|
+
origin_path: publia
|
|
142
|
+
-
|
|
143
|
+
dest_path: publishing_agency_unstem_search
|
|
144
|
+
formatters:
|
|
145
|
+
- CDMDEXER::StripFormatter
|
|
146
|
+
origin_path: publia
|
|
147
|
+
-
|
|
148
|
+
dest_path: publishing_agency_ssi
|
|
149
|
+
formatters:
|
|
150
|
+
- CDMDEXER::StripFormatter
|
|
151
|
+
origin_path: publia
|
|
152
|
+
-
|
|
153
|
+
dest_path: dimensions_ssi
|
|
154
|
+
formatters:
|
|
155
|
+
- CDMDEXER::StripFormatter
|
|
156
|
+
origin_path: dimens
|
|
157
|
+
-
|
|
158
|
+
dest_path: topic_teim
|
|
159
|
+
formatters:
|
|
160
|
+
- CDMDEXER::StripFormatter
|
|
161
|
+
- CDMDEXER::SplitFormatter
|
|
162
|
+
- CDMDEXER::StripFormatter
|
|
163
|
+
origin_path: genera
|
|
164
|
+
-
|
|
165
|
+
dest_path: topic_ssim
|
|
166
|
+
formatters:
|
|
167
|
+
- CDMDEXER::Titlieze
|
|
168
|
+
- CDMDEXER::StripFormatter
|
|
169
|
+
- CDMDEXER::SplitFormatter
|
|
170
|
+
- CDMDEXER::StripFormatter
|
|
171
|
+
origin_path: genera
|
|
172
|
+
-
|
|
173
|
+
dest_path: topic_unstem_search
|
|
174
|
+
formatters:
|
|
175
|
+
- CDMDEXER::StripSemicolonFormatter
|
|
176
|
+
- CDMDEXER::StripFormatter
|
|
177
|
+
origin_path: genera
|
|
178
|
+
-
|
|
179
|
+
dest_path: type_ssi
|
|
180
|
+
formatters:
|
|
181
|
+
- CDMDEXER::Titlieze
|
|
182
|
+
- CDMDEXER::StripSemicolonFormatter
|
|
183
|
+
- CDMDEXER::StripFormatter
|
|
184
|
+
origin_path: type
|
|
185
|
+
-
|
|
186
|
+
dest_path: type_tesi
|
|
187
|
+
formatters:
|
|
188
|
+
- CDMDEXER::Titlieze
|
|
189
|
+
- CDMDEXER::StripSemicolonFormatter
|
|
190
|
+
- CDMDEXER::StripFormatter
|
|
191
|
+
origin_path: type
|
|
192
|
+
-
|
|
193
|
+
dest_path: physical_format_ssi
|
|
194
|
+
formatters:
|
|
195
|
+
- CDMDEXER::StripSemicolonFormatter
|
|
196
|
+
origin_path: physic
|
|
197
|
+
-
|
|
198
|
+
dest_path: physical_format_tesi
|
|
199
|
+
formatters:
|
|
200
|
+
- CDMDEXER::StripSemicolonFormatter
|
|
201
|
+
origin_path: physic
|
|
202
|
+
-
|
|
203
|
+
dest_path: formal_subject_unstem_search
|
|
204
|
+
formatters:
|
|
205
|
+
- CDMDEXER::StripFormatter
|
|
206
|
+
origin_path: specif
|
|
207
|
+
-
|
|
208
|
+
dest_path: formal_subject_ssim
|
|
209
|
+
formatters:
|
|
210
|
+
- CDMDEXER::Titlieze
|
|
211
|
+
- CDMDEXER::StripFormatter
|
|
212
|
+
- CDMDEXER::SplitFormatter
|
|
213
|
+
- CDMDEXER::StripFormatter
|
|
214
|
+
origin_path: specif
|
|
215
|
+
-
|
|
216
|
+
dest_path: formal_subject_teim
|
|
217
|
+
formatters:
|
|
218
|
+
- CDMDEXER::Titlieze
|
|
219
|
+
- CDMDEXER::StripFormatter
|
|
220
|
+
- CDMDEXER::SplitFormatter
|
|
221
|
+
- CDMDEXER::StripFormatter
|
|
222
|
+
origin_path: specif
|
|
223
|
+
-
|
|
224
|
+
dest_path: subject_unstem_search
|
|
225
|
+
formatters:
|
|
226
|
+
- CDMDEXER::StripFormatter
|
|
227
|
+
origin_path: subjec
|
|
228
|
+
-
|
|
229
|
+
dest_path: subject_teim
|
|
230
|
+
formatters:
|
|
231
|
+
- CDMDEXER::StripFormatter
|
|
232
|
+
- CDMDEXER::SplitFormatter
|
|
233
|
+
- CDMDEXER::StripFormatter
|
|
234
|
+
origin_path: subjec
|
|
235
|
+
-
|
|
236
|
+
dest_path: subject_ssim
|
|
237
|
+
formatters:
|
|
238
|
+
- CDMDEXER::StripFormatter
|
|
239
|
+
- CDMDEXER::SplitFormatter
|
|
240
|
+
- CDMDEXER::StripFormatter
|
|
241
|
+
origin_path: subjec
|
|
242
|
+
-
|
|
243
|
+
dest_path: keyword_unstem_search
|
|
244
|
+
formatters:
|
|
245
|
+
- CDMDEXER::KeywordFormatter
|
|
246
|
+
- CDMDEXER::Titlieze
|
|
247
|
+
- CDMDEXER::UniqueFormatter
|
|
248
|
+
- CDMDEXER::StripFormatter
|
|
249
|
+
origin_path: /
|
|
250
|
+
-
|
|
251
|
+
dest_path: keyword_tesi
|
|
252
|
+
formatters:
|
|
253
|
+
- CDMDEXER::KeywordFormatter
|
|
254
|
+
- CDMDEXER::Titlieze
|
|
255
|
+
- CDMDEXER::UniqueFormatter
|
|
256
|
+
- CDMDEXER::JoinFormatter
|
|
257
|
+
- CDMDEXER::StripFormatter
|
|
258
|
+
origin_path: /
|
|
259
|
+
-
|
|
260
|
+
dest_path: keyword_ssim
|
|
261
|
+
formatters:
|
|
262
|
+
- CDMDEXER::KeywordFormatter
|
|
263
|
+
- CDMDEXER::Titlieze
|
|
264
|
+
- CDMDEXER::UniqueFormatter
|
|
265
|
+
- CDMDEXER::StripFormatter
|
|
266
|
+
origin_path: /
|
|
267
|
+
-
|
|
268
|
+
dest_path: city_ssim
|
|
269
|
+
formatters:
|
|
270
|
+
- CDMDEXER::StripFormatter
|
|
271
|
+
- CDMDEXER::SplitFormatter
|
|
272
|
+
- CDMDEXER::StripFormatter
|
|
273
|
+
origin_path: city
|
|
274
|
+
-
|
|
275
|
+
dest_path: city_unstem_search
|
|
276
|
+
formatters:
|
|
277
|
+
- CDMDEXER::StripFormatter
|
|
278
|
+
origin_path: city
|
|
279
|
+
-
|
|
280
|
+
dest_path: district_ssi
|
|
281
|
+
formatters:
|
|
282
|
+
- CDMDEXER::StripFormatter
|
|
283
|
+
origin_path: distri
|
|
284
|
+
-
|
|
285
|
+
dest_path: district_unstem_search
|
|
286
|
+
formatters:
|
|
287
|
+
- CDMDEXER::StripFormatter
|
|
288
|
+
origin_path: distri
|
|
289
|
+
-
|
|
290
|
+
dest_path: county_ssim
|
|
291
|
+
formatters:
|
|
292
|
+
- CDMDEXER::Titlieze
|
|
293
|
+
- CDMDEXER::StripFormatter
|
|
294
|
+
- CDMDEXER::SplitFormatter
|
|
295
|
+
- CDMDEXER::StripFormatter
|
|
296
|
+
origin_path: county
|
|
297
|
+
-
|
|
298
|
+
dest_path: county_unstem_search
|
|
299
|
+
formatters:
|
|
300
|
+
- CDMDEXER::StripFormatter
|
|
301
|
+
origin_path: county
|
|
302
|
+
-
|
|
303
|
+
dest_path: state_ssi
|
|
304
|
+
formatters:
|
|
305
|
+
- CDMDEXER::StripFormatter
|
|
306
|
+
origin_path: state
|
|
307
|
+
-
|
|
308
|
+
dest_path: state_unstem_search
|
|
309
|
+
formatters:
|
|
310
|
+
- CDMDEXER::StripFormatter
|
|
311
|
+
origin_path: state
|
|
312
|
+
-
|
|
313
|
+
dest_path: country_ssi
|
|
314
|
+
formatters:
|
|
315
|
+
- CDMDEXER::StripFormatter
|
|
316
|
+
origin_path: countr
|
|
317
|
+
-
|
|
318
|
+
dest_path: country_unstem_search
|
|
319
|
+
formatters:
|
|
320
|
+
- CDMDEXER::StripFormatter
|
|
321
|
+
origin_path: countr
|
|
322
|
+
-
|
|
323
|
+
dest_path: language_ssi
|
|
324
|
+
formatters:
|
|
325
|
+
- CDMDEXER::StripFormatter
|
|
326
|
+
origin_path: langua
|
|
327
|
+
-
|
|
328
|
+
dest_path: language_unstem_search
|
|
329
|
+
formatters:
|
|
330
|
+
- CDMDEXER::StripFormatter
|
|
331
|
+
origin_path: langua
|
|
332
|
+
-
|
|
333
|
+
dest_path: contributing_unstem_search
|
|
334
|
+
formatters:
|
|
335
|
+
- CDMDEXER::StripFormatter
|
|
336
|
+
origin_path: contra
|
|
337
|
+
-
|
|
338
|
+
dest_path: contributing_organization_tesi
|
|
339
|
+
formatters:
|
|
340
|
+
- CDMDEXER::StripFormatter
|
|
341
|
+
origin_path: contra
|
|
342
|
+
-
|
|
343
|
+
dest_path: contributing_organization_ssi
|
|
344
|
+
formatters:
|
|
345
|
+
- CDMDEXER::Titlieze
|
|
346
|
+
- CDMDEXER::StripFormatter
|
|
347
|
+
origin_path: contra
|
|
348
|
+
-
|
|
349
|
+
dest_path: contact_information_ssi
|
|
350
|
+
formatters:
|
|
351
|
+
- CDMDEXER::StripFormatter
|
|
352
|
+
origin_path: contac
|
|
353
|
+
-
|
|
354
|
+
dest_path: rights_ssi
|
|
355
|
+
formatters:
|
|
356
|
+
- CDMDEXER::StripFormatter
|
|
357
|
+
origin_path: righta
|
|
358
|
+
-
|
|
359
|
+
dest_path: local_identifier_ssi
|
|
360
|
+
formatters:
|
|
361
|
+
- CDMDEXER::StripFormatter
|
|
362
|
+
origin_path: identi
|
|
363
|
+
-
|
|
364
|
+
dest_path: identifier_ssi
|
|
365
|
+
formatters:
|
|
366
|
+
- CDMDEXER::StripFormatter
|
|
367
|
+
origin_path: resour
|
|
368
|
+
-
|
|
369
|
+
dest_path: project_ssi
|
|
370
|
+
formatters:
|
|
371
|
+
- CDMDEXER::StripFormatter
|
|
372
|
+
origin_path: projec
|
|
373
|
+
-
|
|
374
|
+
dest_path: fiscal_sponsor_ssi
|
|
375
|
+
formatters:
|
|
376
|
+
- CDMDEXER::StripFormatter
|
|
377
|
+
origin_path: fiscal
|
|
378
|
+
-
|
|
379
|
+
dest_path: publisher_ssi
|
|
380
|
+
formatters:
|
|
381
|
+
- CDMDEXER::StripFormatter
|
|
382
|
+
origin_path: publis
|
|
383
|
+
-
|
|
384
|
+
dest_path: date_ssi
|
|
385
|
+
formatters:
|
|
386
|
+
- CDMDEXER::StripFormatter
|
|
387
|
+
origin_path: date
|
|
388
|
+
-
|
|
389
|
+
dest_path: format_tesi
|
|
390
|
+
formatters:
|
|
391
|
+
- CDMDEXER::StripFormatter
|
|
392
|
+
origin_path: format
|
|
393
|
+
-
|
|
394
|
+
dest_path: digspa_ssi
|
|
395
|
+
origin_path: digspa
|
|
396
|
+
-
|
|
397
|
+
dest_path: digspb_ssi
|
|
398
|
+
origin_path: digspb
|
|
399
|
+
-
|
|
400
|
+
dest_path: digspc_ssi
|
|
401
|
+
origin_path: digspc
|
|
402
|
+
-
|
|
403
|
+
dest_path: digspd_ssi
|
|
404
|
+
origin_path: digspd
|
|
405
|
+
-
|
|
406
|
+
dest_path: digspe_ssi
|
|
407
|
+
origin_path: digspe
|
|
408
|
+
-
|
|
409
|
+
dest_path: digspf_ssi
|
|
410
|
+
origin_path: digspf
|
|
411
|
+
-
|
|
412
|
+
dest_path: digspg_ssi
|
|
413
|
+
origin_path: digspg
|
|
414
|
+
-
|
|
415
|
+
dest_path: digsph_ssi
|
|
416
|
+
origin_path: digsph
|
|
417
|
+
-
|
|
418
|
+
dest_path: digspi_ssi
|
|
419
|
+
origin_path: digspi
|
|
420
|
+
-
|
|
421
|
+
dest_path: digspj_ssi
|
|
422
|
+
origin_path: digspj
|
|
423
|
+
-
|
|
424
|
+
dest_path: digspk_ssi
|
|
425
|
+
origin_path: digspk
|
|
426
|
+
-
|
|
427
|
+
dest_path: transcription_tesi
|
|
428
|
+
formatters:
|
|
429
|
+
- CDMDEXER::StripFormatter
|
|
430
|
+
origin_path: transc
|
|
431
|
+
-
|
|
432
|
+
dest_path: translation_tesi
|
|
433
|
+
formatters:
|
|
434
|
+
- CDMDEXER::StripFormatter
|
|
435
|
+
origin_path: transl
|
|
436
|
+
-
|
|
437
|
+
dest_path: fullrs_tesi
|
|
438
|
+
formatters:
|
|
439
|
+
- CDMDEXER::StripFormatter
|
|
440
|
+
origin_path: fullrs
|
|
441
|
+
-
|
|
442
|
+
dest_path: find_ssi
|
|
443
|
+
formatters:
|
|
444
|
+
- CDMDEXER::StripFormatter
|
|
445
|
+
origin_path: find
|
|
446
|
+
-
|
|
447
|
+
dest_path: dmaccess_ssi
|
|
448
|
+
formatters:
|
|
449
|
+
- CDMDEXER::StripFormatter
|
|
450
|
+
origin_path: dmaccess
|
|
451
|
+
-
|
|
452
|
+
dest_path: dmimage_ssi
|
|
453
|
+
formatters:
|
|
454
|
+
- CDMDEXER::StripFormatter
|
|
455
|
+
origin_path: dmimage
|
|
456
|
+
-
|
|
457
|
+
dest_path: dmcreated_ssi
|
|
458
|
+
formatters:
|
|
459
|
+
- CDMDEXER::StripFormatter
|
|
460
|
+
origin_path: dmcreated
|
|
461
|
+
-
|
|
462
|
+
dest_path: dmmodified_ssi
|
|
463
|
+
formatters:
|
|
464
|
+
- CDMDEXER::StripFormatter
|
|
465
|
+
origin_path: dmmodified
|
|
466
|
+
-
|
|
467
|
+
dest_path: dmoclcno_ssi
|
|
468
|
+
formatters:
|
|
469
|
+
- CDMDEXER::StripFormatter
|
|
470
|
+
origin_path: dmoclcno
|
|
471
|
+
-
|
|
472
|
+
dest_path: restriction_code_ssi
|
|
473
|
+
formatters:
|
|
474
|
+
- CDMDEXER::StripFormatter
|
|
475
|
+
origin_path: restrictionCode
|
|
476
|
+
-
|
|
477
|
+
dest_path: cdmfilesize_ssi
|
|
478
|
+
formatters:
|
|
479
|
+
- CDMDEXER::StripFormatter
|
|
480
|
+
origin_path: cdmfilesize
|
|
481
|
+
-
|
|
482
|
+
dest_path: cdmfilesizeformatted_ssi
|
|
483
|
+
formatters:
|
|
484
|
+
- CDMDEXER::StripFormatter
|
|
485
|
+
origin_path: cdmfilesizeformatted
|
|
486
|
+
-
|
|
487
|
+
dest_path: cdmprintpdf_is
|
|
488
|
+
formatters:
|
|
489
|
+
- CDMDEXER::ToIFormatter
|
|
490
|
+
origin_path: cdmprintpdf
|
|
491
|
+
-
|
|
492
|
+
dest_path: cdmhasocr_is
|
|
493
|
+
formatters:
|
|
494
|
+
- CDMDEXER::ToIFormatter
|
|
495
|
+
origin_path: cdmhasocr
|
|
496
|
+
-
|
|
497
|
+
dest_path: cdmisnewspaper_is
|
|
498
|
+
formatters:
|
|
499
|
+
- CDMDEXER::ToIFormatter
|
|
500
|
+
origin_path: cdmisnewspaper
|
|
501
|
+
-
|
|
502
|
+
dest_path: image_uri_ssi
|
|
503
|
+
formatters:
|
|
504
|
+
- CDMDEXER::StripFormatter
|
|
505
|
+
origin_path: image_uri
|
|
506
|
+
-
|
|
507
|
+
dest_path: record_type_ssi
|
|
508
|
+
formatters:
|
|
509
|
+
- CDMDEXER::StripFormatter
|
|
510
|
+
origin_path: record_type
|
|
511
|
+
-
|
|
512
|
+
dest_path: geographic_feature_ssim
|
|
513
|
+
formatters:
|
|
514
|
+
- CDMDEXER::Titlieze
|
|
515
|
+
- CDMDEXER::StripFormatter
|
|
516
|
+
- CDMDEXER::SplitFormatter
|
|
517
|
+
- CDMDEXER::StripFormatter
|
|
518
|
+
origin_path: geogra
|
|
519
|
+
-
|
|
520
|
+
dest_path: geographic_feature_teim
|
|
521
|
+
formatters:
|
|
522
|
+
- CDMDEXER::StripFormatter
|
|
523
|
+
origin_path: geogra
|
|
524
|
+
-
|
|
525
|
+
dest_path: geographic_feature_unstem_search
|
|
526
|
+
formatters:
|
|
527
|
+
- CDMDEXER::StripFormatter
|
|
528
|
+
origin_path: geogra
|
|
529
|
+
-
|
|
530
|
+
dest_path: geonam_ssi
|
|
531
|
+
formatters:
|
|
532
|
+
- CDMDEXER::StripFormatter
|
|
533
|
+
origin_path: geonam
|
|
534
|
+
-
|
|
535
|
+
dest_path: kaltura_audio_ssi
|
|
536
|
+
formatters:
|
|
537
|
+
- CDMDEXER::StripFormatter
|
|
538
|
+
origin_path: audio
|
|
539
|
+
-
|
|
540
|
+
dest_path: kaltura_audio_playlist_ssi
|
|
541
|
+
formatters:
|
|
542
|
+
- CDMDEXER::StripFormatter
|
|
543
|
+
origin_path: audioa
|
|
544
|
+
-
|
|
545
|
+
dest_path: kaltura_video_ssi
|
|
546
|
+
formatters:
|
|
547
|
+
- CDMDEXER::StripFormatter
|
|
548
|
+
origin_path: video
|
|
549
|
+
-
|
|
550
|
+
dest_path: kaltura_video_playlist_ssi
|
|
551
|
+
formatters:
|
|
552
|
+
- CDMDEXER::StripFormatter
|
|
553
|
+
origin_path: videoa
|
|
554
|
+
-
|
|
555
|
+
dest_path: coordinates_llsi
|
|
556
|
+
formatters:
|
|
557
|
+
- CDMDEXER::GeoNameID
|
|
558
|
+
- CDMDEXER::GeoNameIDToJson
|
|
559
|
+
- CDMDEXER::GeoNameToLocation
|
|
560
|
+
origin_path: geonam
|
|
561
|
+
-
|
|
562
|
+
dest_path: placename_ssim
|
|
563
|
+
formatters:
|
|
564
|
+
- CDMDEXER::GeoNameID
|
|
565
|
+
- CDMDEXER::GeoNameIDToJson
|
|
566
|
+
- CDMDEXER::GeoNameToPlaceName
|
|
567
|
+
origin_path: geonam
|
|
568
|
+
-
|
|
569
|
+
dest_path: placename_unstem_search
|
|
570
|
+
formatters:
|
|
571
|
+
- CDMDEXER::GeoNameID
|
|
572
|
+
- CDMDEXER::GeoNameIDToJson
|
|
573
|
+
- CDMDEXER::GeoNameToPlaceName
|
|
574
|
+
origin_path: geonam
|
|
575
|
+
-
|
|
576
|
+
dest_path: table_ssim
|
|
577
|
+
formatters:
|
|
578
|
+
- CDMDEXER::StripFormatter
|
|
579
|
+
- CDMDEXER::SplitFormatter
|
|
580
|
+
- CDMDEXER::StripFormatter
|
|
581
|
+
origin_path: table
|
|
582
|
+
-
|
|
583
|
+
dest_path: umedia_ssi
|
|
584
|
+
formatters:
|
|
585
|
+
- CDMDEXER::StripFormatter
|
|
586
|
+
origin_path: umedia
|
|
587
|
+
-
|
|
588
|
+
dest_path: child_index
|
|
589
|
+
formatters: []
|
|
590
|
+
origin_path: child_index
|
metadata
CHANGED
|
@@ -1,43 +1,71 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cdmdexer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.22.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- chadfennell
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-03-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name:
|
|
14
|
+
name: config
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '
|
|
19
|
+
version: '3.1'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '
|
|
26
|
+
version: '3.1'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: contentdm_api
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
31
|
- - "~>"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: 0.
|
|
33
|
+
version: 0.6.0
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: 0.
|
|
40
|
+
version: 0.6.0
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: hash_at_path
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 0.1.6
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 0.1.6
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: rsolr
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '2.0'
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '2.0'
|
|
41
69
|
- !ruby/object:Gem::Dependency
|
|
42
70
|
name: sidekiq
|
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -67,61 +95,75 @@ dependencies:
|
|
|
67
95
|
- !ruby/object:Gem::Version
|
|
68
96
|
version: '1.4'
|
|
69
97
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
98
|
+
name: rails
|
|
71
99
|
requirement: !ruby/object:Gem::Requirement
|
|
72
100
|
requirements:
|
|
73
101
|
- - "~>"
|
|
74
102
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '
|
|
103
|
+
version: '6.0'
|
|
76
104
|
type: :runtime
|
|
77
105
|
prerelease: false
|
|
78
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
107
|
requirements:
|
|
80
108
|
- - "~>"
|
|
81
109
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: '
|
|
110
|
+
version: '6.0'
|
|
83
111
|
- !ruby/object:Gem::Dependency
|
|
84
|
-
name:
|
|
112
|
+
name: rexml
|
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|
|
86
114
|
requirements:
|
|
87
|
-
- - "
|
|
115
|
+
- - "~>"
|
|
88
116
|
- !ruby/object:Gem::Version
|
|
89
|
-
version: '
|
|
117
|
+
version: '3.2'
|
|
90
118
|
type: :runtime
|
|
91
119
|
prerelease: false
|
|
92
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
121
|
requirements:
|
|
94
|
-
- - "
|
|
122
|
+
- - "~>"
|
|
95
123
|
- !ruby/object:Gem::Version
|
|
96
|
-
version: '
|
|
124
|
+
version: '3.2'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: dotenv-rails
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - "~>"
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: 2.7.6
|
|
132
|
+
type: :development
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - "~>"
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: 2.7.6
|
|
97
139
|
- !ruby/object:Gem::Dependency
|
|
98
140
|
name: bundler
|
|
99
141
|
requirement: !ruby/object:Gem::Requirement
|
|
100
142
|
requirements:
|
|
101
143
|
- - "~>"
|
|
102
144
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: '
|
|
145
|
+
version: '2.0'
|
|
104
146
|
type: :development
|
|
105
147
|
prerelease: false
|
|
106
148
|
version_requirements: !ruby/object:Gem::Requirement
|
|
107
149
|
requirements:
|
|
108
150
|
- - "~>"
|
|
109
151
|
- !ruby/object:Gem::Version
|
|
110
|
-
version: '
|
|
152
|
+
version: '2.0'
|
|
111
153
|
- !ruby/object:Gem::Dependency
|
|
112
|
-
name:
|
|
154
|
+
name: byebug
|
|
113
155
|
requirement: !ruby/object:Gem::Requirement
|
|
114
156
|
requirements:
|
|
115
157
|
- - "~>"
|
|
116
158
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
159
|
+
version: '11.1'
|
|
118
160
|
type: :development
|
|
119
161
|
prerelease: false
|
|
120
162
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
163
|
requirements:
|
|
122
164
|
- - "~>"
|
|
123
165
|
- !ruby/object:Gem::Version
|
|
124
|
-
version: '
|
|
166
|
+
version: '11.1'
|
|
125
167
|
- !ruby/object:Gem::Dependency
|
|
126
168
|
name: minitest
|
|
127
169
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -136,6 +178,20 @@ dependencies:
|
|
|
136
178
|
- - "~>"
|
|
137
179
|
- !ruby/object:Gem::Version
|
|
138
180
|
version: '5.0'
|
|
181
|
+
- !ruby/object:Gem::Dependency
|
|
182
|
+
name: rake
|
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
|
184
|
+
requirements:
|
|
185
|
+
- - "~>"
|
|
186
|
+
- !ruby/object:Gem::Version
|
|
187
|
+
version: '12.0'
|
|
188
|
+
type: :development
|
|
189
|
+
prerelease: false
|
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
191
|
+
requirements:
|
|
192
|
+
- - "~>"
|
|
193
|
+
- !ruby/object:Gem::Version
|
|
194
|
+
version: '12.0'
|
|
139
195
|
- !ruby/object:Gem::Dependency
|
|
140
196
|
name: yard
|
|
141
197
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -150,13 +206,15 @@ dependencies:
|
|
|
150
206
|
- - "~>"
|
|
151
207
|
- !ruby/object:Gem::Version
|
|
152
208
|
version: 0.9.0
|
|
153
|
-
description:
|
|
209
|
+
description:
|
|
154
210
|
email:
|
|
155
211
|
- fenne035@umn.edu
|
|
156
212
|
executables: []
|
|
157
213
|
extensions: []
|
|
158
214
|
extra_rdoc_files: []
|
|
159
215
|
files:
|
|
216
|
+
- ".env.example"
|
|
217
|
+
- ".github/workflows/ruby.yml"
|
|
160
218
|
- ".gitignore"
|
|
161
219
|
- ".rubocop.yml"
|
|
162
220
|
- ".travis.yml"
|
|
@@ -197,12 +255,16 @@ files:
|
|
|
197
255
|
- lib/cdmdexer/transformation_error_message.rb
|
|
198
256
|
- lib/cdmdexer/transformer.rb
|
|
199
257
|
- lib/cdmdexer/version.rb
|
|
258
|
+
- lib/generators/cdmdexer/install_generator.rb
|
|
259
|
+
- lib/generators/cdmdexer/settings_generator.rb
|
|
260
|
+
- lib/generators/cdmdexer/templates/cdmdexer.yml
|
|
261
|
+
- lib/generators/cdmdexer/templates/rails_config.rb
|
|
200
262
|
- travis.yml
|
|
201
|
-
homepage:
|
|
263
|
+
homepage:
|
|
202
264
|
licenses:
|
|
203
265
|
- MIT
|
|
204
266
|
metadata: {}
|
|
205
|
-
post_install_message:
|
|
267
|
+
post_install_message:
|
|
206
268
|
rdoc_options: []
|
|
207
269
|
require_paths:
|
|
208
270
|
- lib
|
|
@@ -217,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
217
279
|
- !ruby/object:Gem::Version
|
|
218
280
|
version: '0'
|
|
219
281
|
requirements: []
|
|
220
|
-
rubygems_version: 3.
|
|
221
|
-
signing_key:
|
|
282
|
+
rubygems_version: 3.2.32
|
|
283
|
+
signing_key:
|
|
222
284
|
specification_version: 4
|
|
223
285
|
summary: Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails
|
|
224
286
|
application.
|