hydra-derivatives 3.4.1 → 3.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +59 -0
- data/.solr_wrapper +2 -0
- data/CODE_OF_CONDUCT.md +36 -0
- data/CONTRIBUTING.md +23 -21
- data/Gemfile +3 -1
- data/README.md +79 -6
- data/Rakefile +13 -1
- data/SUPPORT.md +5 -0
- data/VERSION +1 -1
- data/config/fcrepo_wrapper_test.yml +2 -2
- data/config/solr_wrapper_test.yml +2 -0
- data/hydra-derivatives.gemspec +12 -8
- data/lib/hydra/derivatives/config.rb +1 -1
- data/lib/hydra/derivatives/logger.rb +2 -2
- data/lib/hydra/derivatives/processors/document.rb +4 -1
- data/lib/hydra/derivatives/processors/full_text.rb +16 -7
- data/lib/hydra/derivatives/processors/image.rb +4 -2
- data/lib/hydra/derivatives/processors/jpeg2k_image.rb +2 -8
- data/lib/hydra/derivatives/processors/processor.rb +0 -7
- data/lib/hydra/derivatives/processors/shell_based_processor.rb +5 -3
- data/lib/hydra/derivatives/services/persist_output_file_service.rb +1 -1
- data/solr/config/schema.xml +0 -2
- data/solr/config/xslt/example.xsl +1 -1
- data/solr/config/xslt/luke.xsl +1 -1
- metadata +155 -93
- data/.travis.yml +0 -38
- data/spec/fixtures/FlashPix.ppt +0 -0
- data/spec/fixtures/adobe1998.tif +0 -0
- data/spec/fixtures/countdown.avi +0 -0
- data/spec/fixtures/jpeg2k_config.yml +0 -20
- data/spec/fixtures/piano_note.wav +0 -0
- data/spec/fixtures/sample.rtf +0 -68
- data/spec/fixtures/test.dng +0 -0
- data/spec/fixtures/test.doc +0 -0
- data/spec/fixtures/test.docx +0 -0
- data/spec/fixtures/test.pdf +0 -0
- data/spec/fixtures/test.pptx +0 -0
- data/spec/fixtures/test.tif +0 -0
- data/spec/fixtures/test.xls +0 -0
- data/spec/fixtures/test.xlsx +0 -0
- data/spec/fixtures/world.png +0 -0
- data/spec/processors/active_encode_spec.rb +0 -132
- data/spec/processors/document_spec.rb +0 -41
- data/spec/processors/full_text_spec.rb +0 -90
- data/spec/processors/image_spec.rb +0 -124
- data/spec/processors/jpeg2k_spec.rb +0 -82
- data/spec/processors/processor_spec.rb +0 -35
- data/spec/processors/shell_based_processor_spec.rb +0 -19
- data/spec/processors/video_spec.rb +0 -42
- data/spec/runners/active_encode_derivatives_spec.rb +0 -38
- data/spec/services/audio_derivatives_spec.rb +0 -78
- data/spec/services/persist_basic_contained_output_file_service_spec.rb +0 -42
- data/spec/services/persist_external_file_output_file_service_spec.rb +0 -26
- data/spec/services/remote_source_file_spec.rb +0 -33
- data/spec/services/retrieve_source_file_service_spec.rb +0 -60
- data/spec/services/tempfile_service_spec.rb +0 -31
- data/spec/spec_helper.rb +0 -33
- data/spec/units/audio_encoder_spec.rb +0 -23
- data/spec/units/config_spec.rb +0 -50
- data/spec/units/derivatives_spec.rb +0 -64
- data/spec/units/io_decorator_spec.rb +0 -45
- data/spec/units/logger_spec.rb +0 -22
- data/spec/units/transcoding_spec.rb +0 -354
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 506f0f1154a7b2ec1e3af2d55ad7b1a20687c9061303339928e87b42c71d7dd8
|
4
|
+
data.tar.gz: 5af2d8d9e93cc8d8f12d5e5726775721332e0bcb568017b0c486f94c8160bee7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5614a1fb4662d0bac907f0302f811d1850f98a98ade8e70b3bc3377fd2adce97112f6e24bf35d0a017689742b9eccca5276b82300529ddc4d1023fd787ab7fb9
|
7
|
+
data.tar.gz: 43d0a1555fe82600b663cc8b07b3dbc8d8d6e7743b964d31e3a50b1858006870e4076e57d9399855f1eaf95ca729bf41007eac74ca1fe3487809eae841220f33
|
@@ -0,0 +1,59 @@
|
|
1
|
+
version: 2.1
|
2
|
+
orbs:
|
3
|
+
samvera: samvera/circleci-orb@1.0
|
4
|
+
jobs:
|
5
|
+
bundle_lint_test:
|
6
|
+
parameters:
|
7
|
+
ruby_version:
|
8
|
+
type: string
|
9
|
+
bundler_version:
|
10
|
+
type: string
|
11
|
+
default: 2.1.4
|
12
|
+
executor:
|
13
|
+
name: 'samvera/ruby_fcrepo_solr'
|
14
|
+
ruby_version: << parameters.ruby_version >>
|
15
|
+
solr_version: 7-slim
|
16
|
+
steps:
|
17
|
+
- samvera/cached_checkout
|
18
|
+
- run:
|
19
|
+
name: Install dependencies
|
20
|
+
command: |
|
21
|
+
sudo apt-get update
|
22
|
+
sudo apt-get install ghostscript libpng-dev imagemagick ffmpeg libreoffice dcraw
|
23
|
+
- setup_remote_docker
|
24
|
+
- run:
|
25
|
+
name: Install Kakadu
|
26
|
+
command: |
|
27
|
+
docker create --name kakadu josejuansanchez/kakadu:1.0
|
28
|
+
docker cp kakadu:/kakadu kakadu
|
29
|
+
sudo cp kakadu/*.so /usr/lib
|
30
|
+
sudo cp kakadu/* /usr/bin
|
31
|
+
- run:
|
32
|
+
name: Modify ImageMagick security policy
|
33
|
+
command: sudo sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read|write" pattern="PDF"/' /etc/ImageMagick-6/policy.xml
|
34
|
+
- run:
|
35
|
+
name: Modify ImageMagick delegate for raw files
|
36
|
+
command: sudo sed -i 's/decode="dng:decode" command=".*"\/>$/decode="dng:decode" command="\"\;dcraw\"\; -c -q 3 -H 5 -w \"\;\%i\"\; \| \"\;convert\"\; - \"\;\%u\.png\"\;"\/>/' /etc/ImageMagick-6/delegates.xml
|
37
|
+
|
38
|
+
- samvera/bundle:
|
39
|
+
ruby_version: << parameters.ruby_version >>
|
40
|
+
bundler_version: << parameters.bundler_version >>
|
41
|
+
|
42
|
+
- samvera/install_solr_core
|
43
|
+
|
44
|
+
- samvera/rubocop
|
45
|
+
|
46
|
+
- samvera/parallel_rspec
|
47
|
+
|
48
|
+
workflows:
|
49
|
+
ci:
|
50
|
+
jobs:
|
51
|
+
- bundle_lint_test:
|
52
|
+
name: ruby2-5
|
53
|
+
ruby_version: 2.5.9
|
54
|
+
- bundle_lint_test:
|
55
|
+
name: ruby2-6
|
56
|
+
ruby_version: 2.6.9
|
57
|
+
- bundle_lint_test:
|
58
|
+
name: ruby2-7
|
59
|
+
ruby_version: 2.7.5
|
data/.solr_wrapper
ADDED
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
The Samvera community is dedicated to providing a welcoming and
|
2
|
+
positive experience for all its members, whether they are at a formal
|
3
|
+
gathering, in a social setting, or taking part in activities online.
|
4
|
+
The Samvera community welcomes participation from people all over the
|
5
|
+
world and these members bring with them a wide variety of
|
6
|
+
professional, personal and social backgrounds; whatever these may be,
|
7
|
+
we treat colleagues with dignity and respect.
|
8
|
+
|
9
|
+
Community members communicate primarily in English, though for many of
|
10
|
+
them this is not their native language. We therefore strive to express
|
11
|
+
ourselves simply and clearly remembering that unnecessary use of
|
12
|
+
jargon and slang will be a barrier to understanding for many of our
|
13
|
+
colleagues. We are sensitive to the fact that the international
|
14
|
+
nature of the community means that we span many different social norms
|
15
|
+
around language and behaviour and we strive to conduct ourselves,
|
16
|
+
online and in person, in ways that are unlikely to cause offence.
|
17
|
+
|
18
|
+
Samvera conversations are often information-rich and intended to
|
19
|
+
generate discussion and debate. We discuss ideas from a standpoint of
|
20
|
+
mutual respect and reasoned argument.
|
21
|
+
|
22
|
+
Community members work together to promote a respectful and safe
|
23
|
+
community. In the event that someone’s conduct is causing offence or
|
24
|
+
distress, Samvera has a detailed
|
25
|
+
[Anti-Harassment Policy and Protocol](https://wiki.duraspace.org/display/samvera/Anti-Harassment+Policy)
|
26
|
+
which can be applied to address the problem. The first step in dealing
|
27
|
+
with any serious misconduct is to contact a local meeting organizer,
|
28
|
+
the
|
29
|
+
[Samvera community helpers](https://wiki.duraspace.org/display/samvera/Samvera+Community+Helpers)
|
30
|
+
([email](mailto:helpers@samvera.org)), a community member you
|
31
|
+
trust, or the
|
32
|
+
[Samvera Steering Group](https://wiki.duraspace.org/display/samvera/Samvera+Steering+Group+membership)
|
33
|
+
immediately; at Samvera events, these people can be identified by
|
34
|
+
distinctive name badges. The
|
35
|
+
[Policy and Protocol](https://wiki.duraspace.org/display/samvera/Anti-Harassment+Policy)
|
36
|
+
should be consulted for fuller details.
|
data/CONTRIBUTING.md
CHANGED
@@ -1,21 +1,24 @@
|
|
1
1
|
# How to Contribute
|
2
2
|
|
3
|
-
We want your help to make
|
4
|
-
|
3
|
+
We want your help to make the Samvera community great. There are a few guidelines
|
4
|
+
that we need contributors to follow so that we can have a chance of
|
5
|
+
keeping on top of things.
|
5
6
|
|
6
7
|
## Code of Conduct
|
7
8
|
|
8
|
-
The
|
9
|
-
members, whether they are at a formal gathering, in
|
10
|
-
online.
|
11
|
-
for more information.
|
9
|
+
The Samvera Community is dedicated to providing a welcoming and positive
|
10
|
+
experience for all its members, whether they are at a formal gathering, in
|
11
|
+
a social setting, or taking part in activities online. Please see our
|
12
|
+
[Code of Conduct](CODE_OF_CONDUCT.md) for more information.
|
12
13
|
|
13
|
-
##
|
14
|
+
## Samvera Community Intellectual Property Licensing and Ownership
|
14
15
|
|
15
|
-
All code contributors must have an Individual Contributor License Agreement
|
16
|
-
|
16
|
+
All code contributors must have an Individual Contributor License Agreement
|
17
|
+
(iCLA) on file with the Samvera Steering Group. If the contributor works for
|
18
|
+
an institution, the institution must have a Corporate Contributor License
|
19
|
+
Agreement (cCLA) on file.
|
17
20
|
|
18
|
-
https://wiki.duraspace.org/display/
|
21
|
+
https://wiki.duraspace.org/display/samvera/Samvera+Community+Intellectual+Property+Licensing+and+Ownership
|
19
22
|
|
20
23
|
You should also add yourself to the `CONTRIBUTORS.md` file in the root of the project.
|
21
24
|
|
@@ -31,7 +34,7 @@ You should also add yourself to the `CONTRIBUTORS.md` file in the root of the pr
|
|
31
34
|
### Reporting Issues
|
32
35
|
|
33
36
|
* Make sure you have a [GitHub account](https://github.com/signup/free)
|
34
|
-
* Submit a [Github issue](
|
37
|
+
* Submit a [Github issue](https://github.com/samvera/hydra-derivatives/issues/) by:
|
35
38
|
* Clearly describing the issue
|
36
39
|
* Provide a descriptive summary
|
37
40
|
* Explain the expected behavior
|
@@ -48,7 +51,7 @@ You should also add yourself to the `CONTRIBUTORS.md` file in the root of the pr
|
|
48
51
|
* Please avoid working directly on the `master` branch.
|
49
52
|
* You may find the [hub suite of commands](https://github.com/defunkt/hub) helpful
|
50
53
|
* Make sure you have added sufficient tests and documentation for your changes.
|
51
|
-
* Test functionality with RSpec;
|
54
|
+
* Test functionality with RSpec; Test features / UI with Capybara.
|
52
55
|
* Run _all_ the tests to assure nothing else was accidentally broken.
|
53
56
|
|
54
57
|
### Documenting Code
|
@@ -60,15 +63,11 @@ You should also add yourself to the `CONTRIBUTORS.md` file in the root of the pr
|
|
60
63
|
* If you don't know exactly what a bit of code does, it is extra likely that it needs to be documented. Take a stab at it and ask for feedback in your pull request. You can use the 'blame' button on GitHub to identify the original developer of the code and @mention them in your comment.
|
61
64
|
* This work greatly increases the usability of the code base and supports the on-ramping of new committers.
|
62
65
|
* We will all be understanding of one another's time constraints in this area.
|
63
|
-
* YARD examples:
|
64
|
-
* [Hydra::Works::RemoveGenericFile](https://github.com/projecthydra-labs/hydra-works/blob/master/lib/hydra/works/services/generic_work/remove_generic_file.rb)
|
65
|
-
* [ActiveTriples::LocalName::Minter](https://github.com/ActiveTriples/active_triples-local_name/blob/master/lib/active_triples/local_name/minter.rb)
|
66
66
|
* [Getting started with YARD](http://www.rubydoc.info/gems/yard/file/docs/GettingStarted.md)
|
67
67
|
|
68
68
|
### Committing changes
|
69
69
|
|
70
70
|
* Make commits of logical units.
|
71
|
-
* Your commit should include a high level description of your work in HISTORY.textile
|
72
71
|
* Check for unnecessary whitespace with `git diff --check` before committing.
|
73
72
|
* Make sure your commit messages are [well formed](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
|
74
73
|
* If you created an issue, you can close it by including "Closes #issue" in your commit message. See [Github's blog post for more details](https://github.com/blog/1386-closing-issues-via-commit-messages)
|
@@ -103,6 +102,10 @@ You should also add yourself to the `CONTRIBUTORS.md` file in the root of the pr
|
|
103
102
|
long to fit in 72 characters
|
104
103
|
```
|
105
104
|
|
105
|
+
* Make sure you have added the necessary tests for your changes.
|
106
|
+
* Run _all_ the tests to assure nothing else was accidentally broken.
|
107
|
+
* When you are ready to submit a pull request
|
108
|
+
|
106
109
|
### Submitting Changes
|
107
110
|
|
108
111
|
* Read the article ["Using Pull Requests"](https://help.github.com/articles/using-pull-requests) on GitHub.
|
@@ -125,9 +128,7 @@ We adopted [Github's Pull Request Review](https://help.github.com/articles/about
|
|
125
128
|
Common checks that may occur in our repositories:
|
126
129
|
|
127
130
|
1. Travis CI - where our automated tests are running
|
128
|
-
2.
|
129
|
-
3. Approval Required - Github enforces at least one person approve a pull request. Also, all reviewers that have chimed in must approve.
|
130
|
-
4. CodeClimate - is our code remaining healthy (at least according to static code analysis)
|
131
|
+
2. Approval Required - Github enforces at least one person approve a pull request. Also, all reviewers that have chimed in must approve.
|
131
132
|
|
132
133
|
If one or more of the required checks failed (or are incomplete), the code should not be merged (and the UI will not allow it). If all of the checks have passed, then anyone on the project (including the pull request submitter) may merge the code.
|
133
134
|
|
@@ -144,16 +145,17 @@ First, the person contributing the code is putting themselves out there. Be mind
|
|
144
145
|
This is your chance for a mentoring moment of another developer. Take time to give an honest and thorough review of what has changed. Things to consider:
|
145
146
|
|
146
147
|
* Does the commit message explain what is going on?
|
147
|
-
* Does the code changes have tests? _Not all changes need new tests, some changes are
|
148
|
+
* Does the code changes have tests? _Not all changes need new tests, some changes are refactorings_
|
148
149
|
* Do new or changed methods, modules, and classes have documentation?
|
149
150
|
* Does the commit contain more than it should? Are two separate concerns being addressed in one commit?
|
150
151
|
* Does the description of the new/changed specs match your understanding of what the spec is doing?
|
152
|
+
* Did the Travis tests complete successfully?
|
151
153
|
|
152
154
|
If you are uncertain, bring other contributors into the conversation by assigning them as a reviewer.
|
153
155
|
|
154
156
|
# Additional Resources
|
155
157
|
|
156
158
|
* [General GitHub documentation](http://help.github.com/)
|
157
|
-
* [GitHub pull request documentation](
|
159
|
+
* [GitHub pull request documentation](https://help.github.com/articles/about-pull-requests/)
|
158
160
|
* [Pro Git](http://git-scm.com/book) is both a free and excellent book about Git.
|
159
161
|
* [A Git Config for Contributing](http://ndlib.github.io/practices/my-typical-per-project-git-config/)
|
data/Gemfile
CHANGED
@@ -3,9 +3,11 @@ source 'https://rubygems.org'
|
|
3
3
|
# Please see hydra-derivatives.gemspec for dependency information.
|
4
4
|
gemspec
|
5
5
|
|
6
|
+
gem 'sprockets', '~> 3.7'
|
7
|
+
|
6
8
|
group :development, :test do
|
7
|
-
gem 'byebug' unless ENV['TRAVIS']
|
8
9
|
gem 'coveralls'
|
10
|
+
gem 'rspec_junit_formatter'
|
9
11
|
gem 'rubocop', '~> 0.52.0', require: false
|
10
12
|
gem 'rubocop-rspec', require: false
|
11
13
|
gem 'simplecov'
|
data/README.md
CHANGED
@@ -1,11 +1,35 @@
|
|
1
1
|
# hydra-derivatives
|
2
2
|
|
3
|
+
Code:
|
3
4
|
[![Version](https://badge.fury.io/rb/hydra-derivatives.png)](http://badge.fury.io/rb/hydra-derivatives)
|
4
|
-
[![Build Status](https://
|
5
|
-
[![
|
6
|
-
[![Coverage Status](https://coveralls.io/repos/github/projecthydra/hydra-derivatives/badge.svg?branch=master)](https://coveralls.io/github/projecthydra/hydra-derivatives?branch=master)
|
5
|
+
[![Build Status](https://circleci.com/gh/samvera/hydra-derivatives.svg?style=svg)](https://circleci.com/gh/samvera/hydra-derivatives)
|
6
|
+
[![Coverage Status](https://coveralls.io/repos/github/samvera/hydra-derivatives/badge.svg?branch=master)](https://coveralls.io/github/samvera/hydra-derivatives?branch=master)
|
7
7
|
|
8
|
-
|
8
|
+
Docs:
|
9
|
+
[![Contribution Guidelines](http://img.shields.io/badge/CONTRIBUTING-Guidelines-blue.svg)](./CONTRIBUTING.md)
|
10
|
+
[![Apache 2.0 License](http://img.shields.io/badge/APACHE2-license-blue.svg)](./LICENSE.txt)
|
11
|
+
|
12
|
+
Jump In: [![Slack Status](http://slack.samvera.org/badge.svg)](http://slack.samvera.org/)
|
13
|
+
|
14
|
+
# What is hydra-derivatives?
|
15
|
+
|
16
|
+
Derivative generation for Samvera applications.
|
17
|
+
|
18
|
+
## Product Owner & Maintenance
|
19
|
+
|
20
|
+
**hydra-derivatives** is a Core Component of the Samvera community. The documentation for
|
21
|
+
what this means can be found
|
22
|
+
[here](http://samvera.github.io/core_components.html#requirements-for-a-core-component).
|
23
|
+
|
24
|
+
### Product Owner
|
25
|
+
|
26
|
+
[jenlindner](https://github.com/jenlindner)
|
27
|
+
|
28
|
+
# Help
|
29
|
+
|
30
|
+
The Samvera community is here to help. Please see our [support guide](./SUPPORT.md).
|
31
|
+
|
32
|
+
# Getting Started
|
9
33
|
|
10
34
|
If you have an ActiveFedora class like this:
|
11
35
|
```ruby
|
@@ -141,7 +165,7 @@ Just add `gem 'hydra-derivatives'` to your Gemfile.
|
|
141
165
|
* [GhostScript](https://www.ghostscript.com/)
|
142
166
|
* [ImageMagick](http://www.imagemagick.org/)
|
143
167
|
* Kakadu's [kdu_compress](http://www.kakadusoftware.com/) (optional)
|
144
|
-
* [ufraw](http://ufraw.sourceforge.net/)
|
168
|
+
* [ufraw](http://ufraw.sourceforge.net/) or [dcraw](https://dechifro.org/dcraw/)
|
145
169
|
|
146
170
|
To enable LibreOffice, FFMpeg, ImageMagick, FITS support, and kdu_compress support, make sure they are on your path. Most people will put that in their .bash_profile or somewhere similar.
|
147
171
|
|
@@ -171,6 +195,11 @@ MAGICK_TEMPDIR
|
|
171
195
|
```
|
172
196
|
YMMV as to where setting them will take effect in your app; the application's web server's vhost directives are a location known to work with an Apache web server set up.
|
173
197
|
|
198
|
+
ImageMagick by default disables reading and writing of PDFs due to a [vulnerability in ghostscript](https://www.kb.cert.org/vuls/id/332928/). Make sure to install a fixed version of ghostscript and modify ImageMagick's security policy to allow reading and writing PDFs:
|
199
|
+
```
|
200
|
+
sudo sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read|write" pattern="PDF"/' /etc/ImageMagick-6/policy.xml
|
201
|
+
```
|
202
|
+
|
174
203
|
## JPEG2k Directives
|
175
204
|
|
176
205
|
Unlike the other processors, the `Jpeg2kImage` processor does not generally accept arguments that directly (or nearly so) translate to the arguments you would give to the corresponding command line utility.
|
@@ -181,10 +210,54 @@ Instead, each directive may contain these arguments:
|
|
181
210
|
* `:to_srgb` (Boolean) : If `true` and the image is a color image it will map the source image color profile to sRGB. Default: `true`
|
182
211
|
* `:resize` (String) : Geometry; the same syntax as the `Hydra::Derivatives::Image` processor
|
183
212
|
* `:recipe` :
|
184
|
-
- If a Symbol the recipe will be read from the `Hydra::Derivatives.kdu_compress_recipes` hash. You can override this, or a couple of samples are supplied. The symbol in the config file should be the name in the model + `_{quality}`, e.g. `recipe: :default` will look `:default_color` or `:
|
213
|
+
- If a Symbol the recipe will be read from the `Hydra::Derivatives.kdu_compress_recipes` hash. You can override this, or a couple of samples are supplied. The symbol in the config file should be the name in the model + `_{quality}`, e.g. `recipe: :default` will look `:default_color` or `:default_gray` in the hash.
|
185
214
|
- If a String the recipe in the string will be used. You may include anything the command line utility will accept except `-i` or `-o`. See `$ kdu_compress -usage` in your shell.
|
186
215
|
- If no `:recipe` is provided the processor will examine the image and make a best guess, but you can set a few basic options (the remainder of this list). Note that these are ignored if you provided a recipe via either of the first two methods described.
|
187
216
|
* `:levels` (Integer) : The number of decomposition levels. The default is the number of times the long dimension can be divided by two, down to 96, e.g. a 7200 pixel image would have 6 levels (3600, 1800, 900, 450, 225, 112)
|
188
217
|
* `:layers` (Integer) : The number of quality layers. Default: 8
|
189
218
|
* `:compression` (Integer) : The left number of the compression ratio `n:1`, e.g. 12 will apply 12:1 compression. Default: 10.
|
190
219
|
* `:tile_size` (Integer) : Pixel dimension of the tiles. Default: 1024
|
220
|
+
|
221
|
+
# Development Environment
|
222
|
+
|
223
|
+
## Dependencies
|
224
|
+
|
225
|
+
* [ImageMagick](https://www.imagemagick.org)
|
226
|
+
* On a mac, do `brew install imagemagick`
|
227
|
+
* [LibreOffice](https://www.libreoffice.org/)
|
228
|
+
* * On a mac, do `brew cask install libreoffice`
|
229
|
+
* [Kakadu](http://kakadusoftware.com/)
|
230
|
+
* On a mac, extract the file and run the pkg installer therein (don't get distracted by the files called kdu_show)
|
231
|
+
* [Ghostscript](https://www.ghostscript.com/)
|
232
|
+
* On a mac, `brew install ghostscript`
|
233
|
+
* ufraw
|
234
|
+
* On a mac, `brew install ufraw`
|
235
|
+
* dcraw (ufraw alternative)
|
236
|
+
* On a mac, `brew install dcraw`
|
237
|
+
* You will need to modify ImageMagick's delegate for dng files in order to use dcraw.
|
238
|
+
* First find where the deletegates.xml file is located: `identify -list delegate | grep Path`
|
239
|
+
* Edit the delegates.xml file by replacing the dng:decode delegate line with the following:
|
240
|
+
```<delegate decode="dng:decode" command=""dcraw" -c -q 3 -H 5 -w "%i" | "convert" - "%u.png""/>```
|
241
|
+
* libvpx
|
242
|
+
* On a mac, `brew install libvpx`
|
243
|
+
* ffmpeg
|
244
|
+
* On a mac, `brew install ffmpeg`
|
245
|
+
* Ensure `libvpx` is installed first
|
246
|
+
|
247
|
+
## Running Tests
|
248
|
+
|
249
|
+
1. Run tests with `RAILS_ENV=test bundle exec rake ci`
|
250
|
+
|
251
|
+
## Running specific tests
|
252
|
+
|
253
|
+
If you don't want to run the whole suite all at once like CI, do the following:
|
254
|
+
|
255
|
+
1. Run the test servers with `rake derivatives:test_server`
|
256
|
+
2. Run the tests.
|
257
|
+
|
258
|
+
# Acknowledgments
|
259
|
+
|
260
|
+
This software has been developed by and is brought to you by the Samvera community. Learn more at the
|
261
|
+
[Samvera website](http://samvera.org/).
|
262
|
+
|
263
|
+
![Samvera Logo](https://wiki.duraspace.org/download/thumbnails/87459292/samvera-fall-font2-200w.png?version=1&modificationDate=1498550535816&api=v2)
|
data/Rakefile
CHANGED
@@ -20,7 +20,7 @@ namespace :derivatives do
|
|
20
20
|
end
|
21
21
|
|
22
22
|
RSpec::Core::RakeTask.new(:rspec) do |task|
|
23
|
-
task.rspec_opts = "--tag ~requires_kdu_compress" if ENV['
|
23
|
+
task.rspec_opts = "--tag ~requires_kdu_compress" if ENV['CI']
|
24
24
|
end
|
25
25
|
|
26
26
|
desc 'Start up Solr & Fedora and run tests'
|
@@ -29,6 +29,18 @@ namespace :derivatives do
|
|
29
29
|
Rake::Task['derivatives:rspec'].invoke
|
30
30
|
end
|
31
31
|
end
|
32
|
+
|
33
|
+
desc 'Start up test server'
|
34
|
+
task :test_server do
|
35
|
+
ENV["RAILS_ENV"] = "test"
|
36
|
+
with_test_server do
|
37
|
+
puts "Solr: http://localhost:#{ENV["SOLR_TEST_PORT"]}/solr"
|
38
|
+
puts "Fedora: http://localhost:#{ENV["FCREPO_TEST_PORT"]}/rest"
|
39
|
+
while(1) do
|
40
|
+
sleep(1)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
32
44
|
end
|
33
45
|
|
34
46
|
desc 'Run continuous integration build'
|
data/SUPPORT.md
ADDED
@@ -0,0 +1,5 @@
|
|
1
|
+
If you would like to report an issue, first search [the list of issues](https://github.com/samvera/hydra-derivatives/issues/) to see if someone else has already reported it, and then feel free to [create a new issue](https://github.com/samvera/hydra-derivatives/issues/new).
|
2
|
+
|
3
|
+
If you have questions or need help, please email [the Samvera community tech list](https://groups.google.com/forum/#!forum/samvera-tech) or stop by the #dev channel in [the Samvera community Slack team](https://wiki.duraspace.org/pages/viewpage.action?pageId=87460391#Getintouch!-Slack).
|
4
|
+
|
5
|
+
You can learn more about the various Samvera communication channels on the [Get in touch!](https://wiki.duraspace.org/pages/viewpage.action?pageId=87460391) wiki page.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.6.1
|
data/hydra-derivatives.gemspec
CHANGED
@@ -4,27 +4,31 @@ Gem::Specification.new do |spec|
|
|
4
4
|
spec.name = "hydra-derivatives"
|
5
5
|
spec.version = version
|
6
6
|
spec.authors = ["Justin Coyne"]
|
7
|
-
spec.email = ["
|
7
|
+
spec.email = ["jenlindner@gmail.com", "jcoyne85@stanford.edu"]
|
8
8
|
spec.description = "Derivative generation plugin for hydra"
|
9
9
|
spec.summary = "Derivative generation plugin for hydra"
|
10
10
|
spec.license = "APACHE2"
|
11
11
|
spec.homepage = "https://github.com/projecthydra/hydra-derivatives"
|
12
12
|
|
13
|
-
spec.files = `git ls-files`.split(
|
13
|
+
spec.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR).select { |f| File.dirname(f) !~ %r{\A"?spec|test|features\/?} }
|
14
14
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
15
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
16
15
|
spec.require_paths = ["lib"]
|
17
16
|
|
18
|
-
spec.add_development_dependency 'bundler', '~> 1
|
17
|
+
spec.add_development_dependency 'bundler', '~> 2.1'
|
19
18
|
spec.add_development_dependency 'fcrepo_wrapper', '~> 0.2'
|
19
|
+
spec.add_development_dependency 'pry-byebug'
|
20
|
+
spec.add_development_dependency 'rails', '> 5.1', '< 7.0'
|
20
21
|
spec.add_development_dependency 'rake', '~> 10.1'
|
21
22
|
spec.add_development_dependency 'rspec', '~> 3.1'
|
22
|
-
spec.add_development_dependency "solr_wrapper", "~> 0
|
23
|
+
spec.add_development_dependency "solr_wrapper", "~> 2.0"
|
23
24
|
|
24
|
-
spec.add_dependency 'active-fedora', '>= 11.
|
25
|
+
spec.add_dependency 'active-fedora', '>= 11.5.6',
|
26
|
+
'!= 12.0.0', '!= 12.0.1', '!= 12.0.2', '!= 12.0.3', '!= 12.1.0', '!= 12.1.1', '!= 12.2.0', '!= 12.2.1',
|
27
|
+
'!= 13.0.0', '!= 13.1.0', '!= 13.1.1', '!= 13.1.2', '!= 13.1.3', '!= 13.2.0', '!= 13.2.1'
|
25
28
|
spec.add_dependency 'active_encode', '~>0.1'
|
26
|
-
spec.add_dependency '
|
27
|
-
spec.add_dependency '
|
29
|
+
spec.add_dependency 'activemodel', '< 6.1'
|
30
|
+
spec.add_dependency 'activesupport', '>= 4.0', '< 7'
|
31
|
+
spec.add_dependency 'addressable', '~> 2.5'
|
28
32
|
spec.add_dependency 'deprecation'
|
29
33
|
spec.add_dependency 'mime-types', '> 2.0', '< 4.0'
|
30
34
|
spec.add_dependency 'mini_magick', '>= 3.2', '< 5'
|
@@ -57,7 +57,7 @@ module Hydra
|
|
57
57
|
ORGgen_plt=yes
|
58
58
|
ORGtparts=R
|
59
59
|
"Stiles={1024,1024}" ).gsub(/\s+/, " ").strip,
|
60
|
-
|
60
|
+
default_gray: %(-rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171
|
61
61
|
-jp2_space sLUM
|
62
62
|
-double_buffering 10
|
63
63
|
-num_threads 4
|
@@ -11,8 +11,8 @@ module Hydra::Derivatives
|
|
11
11
|
logger.respond_to? method_name
|
12
12
|
end
|
13
13
|
|
14
|
-
def respond_to_missing?(method_name,
|
15
|
-
logger.respond_to_missing
|
14
|
+
def respond_to_missing?(method_name, include_private = false)
|
15
|
+
logger.send(:respond_to_missing?, method_name, include_private)
|
16
16
|
end
|
17
17
|
|
18
18
|
private
|
@@ -1,3 +1,6 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# An abstract class for asyncronous jobs that convert office documents using soffice
|
1
4
|
module Hydra::Derivatives::Processors
|
2
5
|
class Document < Processor
|
3
6
|
include ShellBasedProcessor
|
@@ -23,7 +26,7 @@ module Hydra::Derivatives::Processors
|
|
23
26
|
if directives.fetch(:format) == "jpg"
|
24
27
|
Hydra::Derivatives::Processors::Image.new(converted_file, directives).process
|
25
28
|
else
|
26
|
-
output_file_service.call(File.
|
29
|
+
output_file_service.call(File.open(converted_file, 'rb'), directives)
|
27
30
|
end
|
28
31
|
end
|
29
32
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Hydra::Derivatives::Processors
|
2
4
|
# Extract the full text from the content using Solr's extract handler
|
3
5
|
class FullText < Processor
|
@@ -22,20 +24,27 @@ module Hydra::Derivatives::Processors
|
|
22
24
|
# TODO: this pulls the whole file into memory. We should stream it from Fedora instead
|
23
25
|
# @return [String] the result of calling the extract service
|
24
26
|
def fetch
|
25
|
-
|
26
|
-
req.use_ssl = true if check_for_ssl
|
27
|
-
resp = req.post(uri.to_s, file_content, request_headers)
|
27
|
+
resp = http_request
|
28
28
|
raise "Solr Extract service was unsuccessful. '#{uri}' returned code #{resp.code} for #{source_path}\n#{resp.body}" unless resp.code == '200'
|
29
|
+
|
29
30
|
file_content.rewind if file_content.respond_to?(:rewind)
|
31
|
+
resp.body.force_encoding(resp.type_params['charset']) if resp.type_params['charset']
|
32
|
+
resp.body
|
33
|
+
end
|
30
34
|
|
31
|
-
|
32
|
-
|
35
|
+
# Send the request to the extract service
|
36
|
+
# @return [Net::HttpResponse] the result of calling the extract service
|
37
|
+
def http_request
|
38
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: check_for_ssl) do |http|
|
39
|
+
req = Net::HTTP::Post.new(uri.request_uri, request_headers)
|
40
|
+
req.basic_auth uri.user, uri.password unless uri.password.nil?
|
41
|
+
req.body = file_content
|
42
|
+
http.request req
|
33
43
|
end
|
34
|
-
resp.body
|
35
44
|
end
|
36
45
|
|
37
46
|
def file_content
|
38
|
-
@
|
47
|
+
@file_content ||= File.open(source_path).read
|
39
48
|
end
|
40
49
|
|
41
50
|
# @return [Hash] the request headers to send to the Solr extract service
|
@@ -99,15 +99,9 @@ module Hydra::Derivatives::Processors
|
|
99
99
|
File.unlink(file_path) unless file_path.nil?
|
100
100
|
end
|
101
101
|
|
102
|
-
def encode_file(recipe,
|
102
|
+
def encode_file(recipe, file_path:)
|
103
103
|
output_file = self.class.tmp_file('.jp2')
|
104
|
-
|
105
|
-
self.class.encode(opts[:file_path], recipe, output_file)
|
106
|
-
else
|
107
|
-
Hydra::Derivatives::TempfileService.create(source_file) do |f|
|
108
|
-
self.class.encode(f.path, recipe, output_file)
|
109
|
-
end
|
110
|
-
end
|
104
|
+
self.class.encode(file_path, recipe, output_file)
|
111
105
|
output_file_service.call(File.open(output_file, 'rb'), directives)
|
112
106
|
File.unlink(output_file)
|
113
107
|
end
|
@@ -19,13 +19,6 @@ module Hydra::Derivatives::Processors
|
|
19
19
|
raise "Processor is an abstract class. Implement `process' on #{self.class.name}"
|
20
20
|
end
|
21
21
|
|
22
|
-
# This governs the output key sent to the persist file service
|
23
|
-
# while this is adequate for storing in Fedora, it's not a great name for saving
|
24
|
-
# to the file system.
|
25
|
-
def output_file_id(name)
|
26
|
-
[out_prefix, name].join('_')
|
27
|
-
end
|
28
|
-
|
29
22
|
def output_filename_for(_name)
|
30
23
|
File.basename(source_path)
|
31
24
|
end
|
@@ -62,9 +62,8 @@ module Hydra::Derivatives::Processors
|
|
62
62
|
err_str = ''
|
63
63
|
stdin, stdout, stderr, wait_thr = popen3(command)
|
64
64
|
context[:pid] = wait_thr[:pid]
|
65
|
+
files = [stderr, stdout]
|
65
66
|
stdin.close
|
66
|
-
stdout.close
|
67
|
-
files = [stderr]
|
68
67
|
|
69
68
|
until all_eof?(files)
|
70
69
|
ready = IO.select(files, nil, nil, 60)
|
@@ -82,11 +81,14 @@ module Hydra::Derivatives::Processors
|
|
82
81
|
err_str << data
|
83
82
|
end
|
84
83
|
rescue EOFError
|
85
|
-
|
84
|
+
Hydra::Derivatives::Logger.debug "Caught an eof error in ShellBasedProcessor"
|
86
85
|
# No big deal.
|
87
86
|
end
|
88
87
|
end
|
89
88
|
end
|
89
|
+
|
90
|
+
stdout.close
|
91
|
+
stderr.close
|
90
92
|
exit_status = wait_thr.value
|
91
93
|
|
92
94
|
raise "Unable to execute command \"#{command}\". Exit code: #{exit_status}\nError message: #{err_str}" unless exit_status.success?
|
data/solr/config/schema.xml
CHANGED
@@ -168,7 +168,6 @@
|
|
168
168
|
<fieldType class="solr.TextField" name="textSuggest" positionIncrementGap="100">
|
169
169
|
<analyzer>
|
170
170
|
<tokenizer class="solr.KeywordTokenizerFactory"/>
|
171
|
-
<filter class="solr.StandardFilterFactory"/>
|
172
171
|
<filter class="solr.LowerCaseFilterFactory"/>
|
173
172
|
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
|
174
173
|
</analyzer>
|
@@ -338,7 +337,6 @@
|
|
338
337
|
<!-- <defaultSearchField>text</defaultSearchField> -->
|
339
338
|
|
340
339
|
<!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
|
341
|
-
<solrQueryParser defaultOperator="OR"/>
|
342
340
|
|
343
341
|
<!-- copyField commands copy one field to another at the time a document
|
344
342
|
is added to the index. It's used either to index the same field differently,
|
@@ -124,7 +124,7 @@
|
|
124
124
|
td { vertical-align: top; }
|
125
125
|
ul { margin: 0px; margin-left: 1em; padding: 0px; }
|
126
126
|
.note { font-size:80%; }
|
127
|
-
.doc { margin-top: 1em; border-top: solid
|
127
|
+
.doc { margin-top: 1em; border-top: solid gray 1px; }
|
128
128
|
.exp { display: none; font-family: monospace; white-space: pre; }
|
129
129
|
</style>
|
130
130
|
</xsl:template>
|