pushmi_pullyu 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.coveralls.yml +1 -0
- data/.editorconfig +16 -0
- data/.gitignore +15 -0
- data/.hound.yml +2 -0
- data/.rspec +2 -0
- data/.rubocop.yml +90 -0
- data/.travis.yml +16 -0
- data/Dangerfile +22 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +135 -0
- data/Rakefile +11 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/config/.gitkeep +0 -0
- data/docs/images/pushmi-pullyu.png +0 -0
- data/docs/images/system-infrastructure-diagram.png +0 -0
- data/examples/pushmi_pullyu.yml +40 -0
- data/exe/pushmi_pullyu +7 -0
- data/lib/pushmi_pullyu.rb +95 -0
- data/lib/pushmi_pullyu/aip.rb +22 -0
- data/lib/pushmi_pullyu/aip/creator.rb +44 -0
- data/lib/pushmi_pullyu/aip/downloader.rb +177 -0
- data/lib/pushmi_pullyu/aip/fedora_fetcher.rb +59 -0
- data/lib/pushmi_pullyu/aip/solr_fetcher.rb +33 -0
- data/lib/pushmi_pullyu/cli.rb +255 -0
- data/lib/pushmi_pullyu/logging.rb +70 -0
- data/lib/pushmi_pullyu/preservation_queue.rb +76 -0
- data/lib/pushmi_pullyu/swift_depositer.rb +51 -0
- data/lib/pushmi_pullyu/version.rb +3 -0
- data/pushmi_pullyu.gemspec +48 -0
- metadata +391 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 392efadaf852092c534f920017037957bb22436c
|
4
|
+
data.tar.gz: e297ea545d6c31ad594721d9c952e9e6a0ccf719
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 751a8b7ce779203c369a41680cb45b394cb67062ff6fdf9c1e944015633ff3feeef50992a8a0b6ee7cc764c270bacad76d407c5b5c9737479417d64b7c6dab57
|
7
|
+
data.tar.gz: 1e2343f0960560cdfc512ddc5a447b1dd80cafd6efae98b16bb39937d01a4798a4f49eb33ce9a7174c23f8d83d8af1578c9b2f889d29593265cff8bff891f56c
|
data/.coveralls.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
service_name: travis-ci
|
data/.editorconfig
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# EditorConfig helps developers define and maintain consistent
|
2
|
+
# coding styles between different editors and IDEs
|
3
|
+
# editorconfig.org
|
4
|
+
|
5
|
+
root = true
|
6
|
+
|
7
|
+
[*]
|
8
|
+
charset = utf-8
|
9
|
+
end_of_line = lf
|
10
|
+
indent_size = 2
|
11
|
+
indent_style = space
|
12
|
+
insert_final_newline = true
|
13
|
+
trim_trailing_whitespace = true
|
14
|
+
|
15
|
+
[*.{diff,md}]
|
16
|
+
trim_trailing_whitespace = false
|
data/.gitignore
ADDED
data/.hound.yml
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
# List of cops can be found here:
|
2
|
+
# https://github.com/bbatsov/rubocop/blob/master/config/default.yml
|
3
|
+
# https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
|
4
|
+
# https://github.com/bbatsov/rubocop/blob/master/config/enabled.yml
|
5
|
+
require: rubocop-rspec
|
6
|
+
|
7
|
+
AllCops:
|
8
|
+
DisplayCopNames: true
|
9
|
+
DisplayStyleGuide: true
|
10
|
+
Exclude:
|
11
|
+
- 'tmp/**/*'
|
12
|
+
- 'vendor/**/*'
|
13
|
+
ExtraDetails: true
|
14
|
+
TargetRubyVersion: 2.3
|
15
|
+
|
16
|
+
# readability is Actually Good
|
17
|
+
Layout/EmptyLinesAroundClassBody:
|
18
|
+
EnforcedStyle: empty_lines_except_namespace
|
19
|
+
|
20
|
+
Layout/IndentationConsistency:
|
21
|
+
Enabled: true
|
22
|
+
EnforcedStyle: normal
|
23
|
+
|
24
|
+
# A calculated magnitude based on number of assignments,
|
25
|
+
# branches, and conditions.
|
26
|
+
Metrics/AbcSize:
|
27
|
+
Enabled: false
|
28
|
+
|
29
|
+
Metrics/ClassLength:
|
30
|
+
Max: 500 # default 100
|
31
|
+
|
32
|
+
# A complexity metric that is strongly correlated to the number
|
33
|
+
# of test cases needed to validate a method.
|
34
|
+
Metrics/CyclomaticComplexity:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
Metrics/LineLength:
|
38
|
+
Enabled: true
|
39
|
+
Max: 120 # default is 80
|
40
|
+
|
41
|
+
# Avoid methods longer than 10 lines of code.
|
42
|
+
Metrics/MethodLength:
|
43
|
+
Enabled: false
|
44
|
+
|
45
|
+
Metrics/BlockLength:
|
46
|
+
Enabled: false
|
47
|
+
|
48
|
+
Metrics/ModuleLength:
|
49
|
+
Max: 500 # default 100
|
50
|
+
|
51
|
+
# A complexity metric geared towards measuring complexity for a human reader.
|
52
|
+
Metrics/PerceivedComplexity:
|
53
|
+
Enabled: false
|
54
|
+
|
55
|
+
# indentation is an endangered resource
|
56
|
+
Style/ClassAndModuleChildren:
|
57
|
+
EnforcedStyle: compact
|
58
|
+
|
59
|
+
Style/Documentation:
|
60
|
+
Enabled: false
|
61
|
+
|
62
|
+
Style/FileName:
|
63
|
+
Exclude:
|
64
|
+
- Dangerfile
|
65
|
+
- Rakefile
|
66
|
+
- Gemfile
|
67
|
+
|
68
|
+
# Checks if there is a magic comment to enforce string literals
|
69
|
+
Style/FrozenStringLiteralComment:
|
70
|
+
Enabled: false
|
71
|
+
|
72
|
+
# Perfer to use // but %r can be better in certain scenarios
|
73
|
+
Style/RegexpLiteral:
|
74
|
+
Enabled: false
|
75
|
+
|
76
|
+
Style/SymbolArray:
|
77
|
+
Enabled: false
|
78
|
+
|
79
|
+
# Use %w or %W for arrays of words.
|
80
|
+
Style/WordArray:
|
81
|
+
Enabled: false
|
82
|
+
|
83
|
+
RSpec/ExampleLength:
|
84
|
+
Enabled: false
|
85
|
+
|
86
|
+
RSpec/MultipleExpectations:
|
87
|
+
Enabled: false
|
88
|
+
|
89
|
+
RSpec/DescribedClass:
|
90
|
+
EnforcedStyle: explicit
|
data/.travis.yml
ADDED
data/Dangerfile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# Make sure non-trivial amounts of code changes come with corresponding tests
|
2
|
+
has_app_changes = !git.modified_files.grep(/lib/).empty?
|
3
|
+
has_spec_changes = !git.modified_files.grep(/spec/).empty?
|
4
|
+
|
5
|
+
if git.lines_of_code > 50 && has_app_changes && !has_spec_changes
|
6
|
+
warn('There are code changes, but no corresponding tests. '\
|
7
|
+
'Please include tests if this PR introduces any modifications in '\
|
8
|
+
'behavior.',
|
9
|
+
sticky: false)
|
10
|
+
end
|
11
|
+
|
12
|
+
# Mainly to encourage writing up some reasoning about the PR, rather than
|
13
|
+
# just leaving a title
|
14
|
+
warn('Please add a detailed summary in the description.') if github.pr_body.length < 5
|
15
|
+
|
16
|
+
# Ensure a clean commits history
|
17
|
+
if git.commits.any? { |c| c.message =~ /^Merge branch '#{github.branch_for_base}'/ }
|
18
|
+
warn 'Please rebase to get rid of the merge commits in this PR'
|
19
|
+
end
|
20
|
+
|
21
|
+
# Warn when there is a big PR
|
22
|
+
warn('This PR is too big! Consider breaking it down into smaller PRs.') if git.lines_of_code > 500
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 University of Alberta Libraries
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
<p align="center">
|
2
|
+
<a href="https://github.com/ualbertalib/pushmi_pullyu/">
|
3
|
+
<img src="docs/images/pushmi-pullyu.png" alt="PushmiPullyu logo">
|
4
|
+
</a>
|
5
|
+
</p>
|
6
|
+
|
7
|
+
[](https://rubygems.org/gems/pushmi_pullyu)
|
8
|
+
[](https://gemnasium.com/github.com/ualbertalib/pushmi_pullyu)
|
9
|
+
[](https://travis-ci.org/ualbertalib/pushmi_pullyu)
|
10
|
+
[](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
|
11
|
+
|
12
|
+
PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
|
13
|
+
|
14
|
+
Its primary job is to manage the flow of content from Fedora into Swift for preservation.
|
15
|
+
|
16
|
+

|
17
|
+
|
18
|
+
## Workflow
|
19
|
+
|
20
|
+
1. Any save (create or update) on a GenericFile in ERA will trigger an after save callback that will push the GenericFile unique identifier (NOID) into a Queue.
|
21
|
+
2. The queue (Redis) is setup to be a unique set (which only allows one GenericFile NOID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
|
22
|
+
3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
|
23
|
+
4. All the GenericFile information and data required for preservation are retrieved from Fedora and Solr using multiple REST calls.
|
24
|
+
5. An Archival Information Package (AIP) is created from the GenericFile's information. It is then bagged and tarred.
|
25
|
+
6. The AIP tar is then uploaded to Swift via a REST call.
|
26
|
+
7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
|
27
|
+
|
28
|
+
|
29
|
+
## Requirements
|
30
|
+
|
31
|
+
PushmiPullyu supports Ruby 2.3.1+
|
32
|
+
|
33
|
+
## Installation
|
34
|
+
|
35
|
+
PushmiPullyu is hosted on rubygems.org. Therefore it can be installed via:
|
36
|
+
|
37
|
+
```bash
|
38
|
+
gem install pushmi_pullyu
|
39
|
+
```
|
40
|
+
|
41
|
+
## Usage
|
42
|
+
|
43
|
+
Foreground mode:
|
44
|
+
```bash
|
45
|
+
pushmi_pullyu
|
46
|
+
```
|
47
|
+
|
48
|
+
Run as a daemon:
|
49
|
+
```bash
|
50
|
+
pushmi_pullyu start
|
51
|
+
```
|
52
|
+
|
53
|
+
PushmiPullyu is quite flexible in that many options can be specified at the command line.
|
54
|
+
Use `pushmi_pullyu --help` to see all the command line options available.
|
55
|
+
|
56
|
+
Example:
|
57
|
+
|
58
|
+
```
|
59
|
+
Usage: pushmi_pullyu [options] [start|stop|restart|run]
|
60
|
+
|
61
|
+
Specific options:
|
62
|
+
-a, --minimum-age AGE Minimum amount of time an item must spend in the queue, in seconds.
|
63
|
+
-d, --debug Enable debug logging
|
64
|
+
-r, --rollbar-token TOKEN Enable error reporting to Rollbar
|
65
|
+
-C, --config PATH Path for YAML config file
|
66
|
+
-L, --logdir PATH Path for directory to store log files
|
67
|
+
-D, --piddir PATH Path for directory to store pid files
|
68
|
+
-W, --workdir PATH Path for directory where AIP creation work takes place in
|
69
|
+
-N, --process_name NAME Name of the application process
|
70
|
+
-m, --monitor Start monitor process for a deamon
|
71
|
+
-q, --queue NAME Name of the queue to read from
|
72
|
+
|
73
|
+
Common options:
|
74
|
+
-v, --version Show version
|
75
|
+
-h, --help Show this message
|
76
|
+
```
|
77
|
+
|
78
|
+
## Configuration file
|
79
|
+
|
80
|
+
You can also provide a configuration file which PushmiPullyu will use with the -C (or --config) flag:
|
81
|
+
|
82
|
+
```bash
|
83
|
+
pushmi_pullyu start -C /path/to/config.yml
|
84
|
+
```
|
85
|
+
|
86
|
+
By default, if no configuration file is specified, PushmiPullyu will look for a configuration file at config/pushmi_pullyu.yml.
|
87
|
+
|
88
|
+
Take the following [sample configuration](https://github.com/ualbertalib/pushmi_pullyu/blob/master/examples/pushmi_pullyu.yml) as inspiration or check out the [DEFAULTS hash](https://github.com/ualbertalib/pushmi_pullyu/blob/master/lib/pushmi_pullyu.rb) in the main module to see all available options.
|
89
|
+
|
90
|
+
Options passed on the command line will also override options specified in the config file.
|
91
|
+
|
92
|
+
## Signals
|
93
|
+
|
94
|
+
PushmiPullyu responds to these signals:
|
95
|
+
|
96
|
+
* `INT`: Equivalent of sending Ctrl-C to PushmiPullyu. Will attempt to finish then exit.
|
97
|
+
* `TERM`: Similar to `INT`. PushmiPullyu will attempt to finish then exit.
|
98
|
+
* `HUP`: Reopens log files as defined in logfile configuration options.
|
99
|
+
|
100
|
+
## Development
|
101
|
+
|
102
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
103
|
+
|
104
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
105
|
+
|
106
|
+
## Testing
|
107
|
+
|
108
|
+
To run the test suite:
|
109
|
+
|
110
|
+
```bash
|
111
|
+
bundle install
|
112
|
+
bundle exec rake
|
113
|
+
```
|
114
|
+
|
115
|
+
This will run both rspec and rubocop together.
|
116
|
+
|
117
|
+
To run rspec by itself:
|
118
|
+
|
119
|
+
```bash
|
120
|
+
bundle exec rspec
|
121
|
+
```
|
122
|
+
To run rubocop by itself:
|
123
|
+
|
124
|
+
```bash
|
125
|
+
bundle exec rubocop
|
126
|
+
```
|
127
|
+
|
128
|
+
## Deployment
|
129
|
+
|
130
|
+
TODO: Fill me out
|
131
|
+
|
132
|
+
## License
|
133
|
+
|
134
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
135
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'pushmi_pullyu'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require 'irb'
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/config/.gitkeep
ADDED
File without changes
|
Binary file
|
Binary file
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# Sample configuration file for PushmiPullyu.
|
2
|
+
# Options here can still be overridden by cmd line args.
|
3
|
+
|
4
|
+
# Pass this file at config/pushmi_pullyu.yml and PushmiPullyu will
|
5
|
+
# pick it up automatically. Or pass this file into pushmi_pullyu on command line
|
6
|
+
# via `pushmi_pullyu -C path/to/pushmi_pullyu.yml`
|
7
|
+
|
8
|
+
# PushmiPullyu will run this file through ERB when reading it so you can
|
9
|
+
# even put in dynamic logic, like consuming ENV Variables.
|
10
|
+
|
11
|
+
debug: false
|
12
|
+
logdir: log
|
13
|
+
monitor: false
|
14
|
+
piddir: tmp/pids
|
15
|
+
workdir: tmp/work
|
16
|
+
process_name: pushmi_pullyu
|
17
|
+
queue_name: dev:pmpy_queue
|
18
|
+
minimum_age: 0
|
19
|
+
rollbar_token: 'abc123xyz'
|
20
|
+
|
21
|
+
redis:
|
22
|
+
host: localhost
|
23
|
+
port: 6379
|
24
|
+
|
25
|
+
solr:
|
26
|
+
url: http://localhost:8983/solr/development
|
27
|
+
|
28
|
+
fedora:
|
29
|
+
url: http://localhost:8983/fedora/rest
|
30
|
+
user: fedoraAdmin
|
31
|
+
password: fedoraAdmin
|
32
|
+
base_path: /dev
|
33
|
+
|
34
|
+
swift:
|
35
|
+
auth_version: v1.0
|
36
|
+
tenant: tester
|
37
|
+
username: test:tester
|
38
|
+
password: testing
|
39
|
+
endpoint: http://localhost:8080
|
40
|
+
container: ERA
|
data/exe/pushmi_pullyu
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'pushmi_pullyu/logging'
|
2
|
+
|
3
|
+
require 'pushmi_pullyu/aip'
|
4
|
+
require 'pushmi_pullyu/aip/creator'
|
5
|
+
require 'pushmi_pullyu/aip/downloader'
|
6
|
+
require 'pushmi_pullyu/aip/solr_fetcher'
|
7
|
+
require 'pushmi_pullyu/aip/fedora_fetcher'
|
8
|
+
require 'pushmi_pullyu/cli'
|
9
|
+
require 'pushmi_pullyu/preservation_queue'
|
10
|
+
require 'pushmi_pullyu/swift_depositer'
|
11
|
+
require 'pushmi_pullyu/version'
|
12
|
+
|
13
|
+
require 'active_support'
|
14
|
+
require 'active_support/core_ext'
|
15
|
+
|
16
|
+
# PushmiPullyu main module
|
17
|
+
module PushmiPullyu
|
18
|
+
DEFAULTS = {
|
19
|
+
daemonize: false,
|
20
|
+
debug: false,
|
21
|
+
logdir: 'log',
|
22
|
+
minimum_age: 0,
|
23
|
+
monitor: false,
|
24
|
+
piddir: 'tmp/pids',
|
25
|
+
workdir: 'tmp/work',
|
26
|
+
process_name: 'pushmi_pullyu',
|
27
|
+
queue_name: 'dev:pmpy_queue',
|
28
|
+
redis: {
|
29
|
+
host: 'localhost',
|
30
|
+
port: 6379
|
31
|
+
},
|
32
|
+
# TODO: rest of these are examples for solr/fedora/swift... feel free to fill them in correctly
|
33
|
+
solr: {
|
34
|
+
url: 'http://localhost:8983/solr/development'
|
35
|
+
},
|
36
|
+
fedora: {
|
37
|
+
url: 'http://localhost:8983/fedora/rest',
|
38
|
+
user: 'fedoraAdmin',
|
39
|
+
password: 'fedoraAdmin',
|
40
|
+
base_path: '/dev'
|
41
|
+
},
|
42
|
+
swift: {
|
43
|
+
auth_version: 'v1.0',
|
44
|
+
tenant: 'tester',
|
45
|
+
username: 'test:tester',
|
46
|
+
password: 'testing',
|
47
|
+
endpoint: 'http://localhost:8080',
|
48
|
+
container: 'ERA'
|
49
|
+
}
|
50
|
+
}.freeze
|
51
|
+
|
52
|
+
def self.options
|
53
|
+
@options ||= DEFAULTS.dup
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.options=(opts)
|
57
|
+
options.deep_merge!(opts)
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.override_options(opts)
|
61
|
+
@options = opts
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.application_log_file
|
65
|
+
"#{options[:logdir]}/#{options[:process_name]}.log"
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.logger
|
69
|
+
PushmiPullyu::Logging.logger
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.logger=(log)
|
73
|
+
PushmiPullyu::Logging.logger = log
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.server_running=(status)
|
77
|
+
@server_running = status
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.reset_logger=(status)
|
81
|
+
@reset_logger = status
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.server_running?
|
85
|
+
@server_running
|
86
|
+
end
|
87
|
+
|
88
|
+
def self.continue_polling?
|
89
|
+
server_running? && !reset_logger?
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.reset_logger?
|
93
|
+
@reset_logger
|
94
|
+
end
|
95
|
+
end
|