pushmi_pullyu 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 392efadaf852092c534f920017037957bb22436c
4
+ data.tar.gz: e297ea545d6c31ad594721d9c952e9e6a0ccf719
5
+ SHA512:
6
+ metadata.gz: 751a8b7ce779203c369a41680cb45b394cb67062ff6fdf9c1e944015633ff3feeef50992a8a0b6ee7cc764c270bacad76d407c5b5c9737479417d64b7c6dab57
7
+ data.tar.gz: 1e2343f0960560cdfc512ddc5a447b1dd80cafd6efae98b16bb39937d01a4798a4f49eb33ce9a7174c23f8d83d8af1578c9b2f889d29593265cff8bff891f56c
@@ -0,0 +1 @@
1
+ service_name: travis-ci
@@ -0,0 +1,16 @@
1
+ # EditorConfig helps developers define and maintain consistent
2
+ # coding styles between different editors and IDEs
3
+ # editorconfig.org
4
+
5
+ root = true
6
+
7
+ [*]
8
+ charset = utf-8
9
+ end_of_line = lf
10
+ indent_size = 2
11
+ indent_style = space
12
+ insert_final_newline = true
13
+ trim_trailing_whitespace = true
14
+
15
+ [*.{diff,md}]
16
+ trim_trailing_whitespace = false
@@ -0,0 +1,15 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /log/
10
+ /tmp/
11
+ /vendor/bundle
12
+ *.gem
13
+
14
+ # rspec failure tracking
15
+ .rspec_status
@@ -0,0 +1,2 @@
1
+ ruby:
2
+ config_file: .rubocop.yml
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
@@ -0,0 +1,90 @@
1
+ # List of cops can be found here:
2
+ # https://github.com/bbatsov/rubocop/blob/master/config/default.yml
3
+ # https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
4
+ # https://github.com/bbatsov/rubocop/blob/master/config/enabled.yml
5
+ require: rubocop-rspec
6
+
7
+ AllCops:
8
+ DisplayCopNames: true
9
+ DisplayStyleGuide: true
10
+ Exclude:
11
+ - 'tmp/**/*'
12
+ - 'vendor/**/*'
13
+ ExtraDetails: true
14
+ TargetRubyVersion: 2.3
15
+
16
+ # readability is Actually Good
17
+ Layout/EmptyLinesAroundClassBody:
18
+ EnforcedStyle: empty_lines_except_namespace
19
+
20
+ Layout/IndentationConsistency:
21
+ Enabled: true
22
+ EnforcedStyle: normal
23
+
24
+ # A calculated magnitude based on number of assignments,
25
+ # branches, and conditions.
26
+ Metrics/AbcSize:
27
+ Enabled: false
28
+
29
+ Metrics/ClassLength:
30
+ Max: 500 # default 100
31
+
32
+ # A complexity metric that is strongly correlated to the number
33
+ # of test cases needed to validate a method.
34
+ Metrics/CyclomaticComplexity:
35
+ Enabled: false
36
+
37
+ Metrics/LineLength:
38
+ Enabled: true
39
+ Max: 120 # default is 80
40
+
41
+ # Avoid methods longer than 10 lines of code.
42
+ Metrics/MethodLength:
43
+ Enabled: false
44
+
45
+ Metrics/BlockLength:
46
+ Enabled: false
47
+
48
+ Metrics/ModuleLength:
49
+ Max: 500 # default 100
50
+
51
+ # A complexity metric geared towards measuring complexity for a human reader.
52
+ Metrics/PerceivedComplexity:
53
+ Enabled: false
54
+
55
+ # indentation is an endangered resource
56
+ Style/ClassAndModuleChildren:
57
+ EnforcedStyle: compact
58
+
59
+ Style/Documentation:
60
+ Enabled: false
61
+
62
+ Style/FileName:
63
+ Exclude:
64
+ - Dangerfile
65
+ - Rakefile
66
+ - Gemfile
67
+
68
+ # Checks if there is a magic comment to enforce string literals
69
+ Style/FrozenStringLiteralComment:
70
+ Enabled: false
71
+
72
+ # Perfer to use // but %r can be better in certain scenarios
73
+ Style/RegexpLiteral:
74
+ Enabled: false
75
+
76
+ Style/SymbolArray:
77
+ Enabled: false
78
+
79
+ # Use %w or %W for arrays of words.
80
+ Style/WordArray:
81
+ Enabled: false
82
+
83
+ RSpec/ExampleLength:
84
+ Enabled: false
85
+
86
+ RSpec/MultipleExpectations:
87
+ Enabled: false
88
+
89
+ RSpec/DescribedClass:
90
+ EnforcedStyle: explicit
@@ -0,0 +1,16 @@
1
+ language: ruby
2
+ sudo: false
3
+ cache: bundler
4
+ services:
5
+ - redis-server
6
+
7
+ before_install:
8
+ - gem install bundler
9
+ - gem update bundler
10
+
11
+ rvm:
12
+ - 2.3.3
13
+ - 2.4.0
14
+
15
+ before_script:
16
+ - bundle exec danger
@@ -0,0 +1,22 @@
1
+ # Make sure non-trivial amounts of code changes come with corresponding tests
2
+ has_app_changes = !git.modified_files.grep(/lib/).empty?
3
+ has_spec_changes = !git.modified_files.grep(/spec/).empty?
4
+
5
+ if git.lines_of_code > 50 && has_app_changes && !has_spec_changes
6
+ warn('There are code changes, but no corresponding tests. '\
7
+ 'Please include tests if this PR introduces any modifications in '\
8
+ 'behavior.',
9
+ sticky: false)
10
+ end
11
+
12
+ # Mainly to encourage writing up some reasoning about the PR, rather than
13
+ # just leaving a title
14
+ warn('Please add a detailed summary in the description.') if github.pr_body.length < 5
15
+
16
+ # Ensure a clean commits history
17
+ if git.commits.any? { |c| c.message =~ /^Merge branch '#{github.branch_for_base}'/ }
18
+ warn 'Please rebase to get rid of the merge commits in this PR'
19
+ end
20
+
21
+ # Warn when there is a big PR
22
+ warn('This PR is too big! Consider breaking it down into smaller PRs.') if git.lines_of_code > 500
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in pushmi_pullyu.gemspec
4
+ gemspec
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 University of Alberta Libraries
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,135 @@
1
+ <p align="center">
2
+ <a href="https://github.com/ualbertalib/pushmi_pullyu/">
3
+ <img src="docs/images/pushmi-pullyu.png" alt="PushmiPullyu logo">
4
+ </a>
5
+ </p>
6
+
7
+ [![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
8
+ [![Dependency Status](https://gemnasium.com/badges/github.com/ualbertalib/pushmi_pullyu.svg)](https://gemnasium.com/github.com/ualbertalib/pushmi_pullyu)
9
+ [![Build Status](https://travis-ci.org/ualbertalib/pushmi_pullyu.svg?branch=master)](https://travis-ci.org/ualbertalib/pushmi_pullyu)
10
+ [![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
11
+
12
+ PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
13
+
14
+ Its primary job is to manage the flow of content from Fedora into Swift for preservation.
15
+
16
+ ![System Infrastructure Diagram](docs/images/system-infrastructure-diagram.png)
17
+
18
+ ## Workflow
19
+
20
+ 1. Any save (create or update) on a GenericFile in ERA will trigger an after save callback that will push the GenericFile unique identifier (NOID) into a Queue.
21
+ 2. The queue (Redis) is setup to be a unique set (which only allows one GenericFile NOID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
22
+ 3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
23
+ 4. All the GenericFile information and data required for preservation are retrieved from Fedora and Solr using multiple REST calls.
24
+ 5. An Archival Information Package (AIP) is created from the GenericFile's information. It is then bagged and tarred.
25
+ 6. The AIP tar is then uploaded to Swift via a REST call.
26
+ 7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
27
+
28
+
29
+ ## Requirements
30
+
31
+ PushmiPullyu supports Ruby 2.3.1+
32
+
33
+ ## Installation
34
+
35
+ PushmiPullyu is hosted on rubygems.org. Therefore it can be installed via:
36
+
37
+ ```bash
38
+ gem install pushmi_pullyu
39
+ ```
40
+
41
+ ## Usage
42
+
43
+ Foreground mode:
44
+ ```bash
45
+ pushmi_pullyu
46
+ ```
47
+
48
+ Run as a daemon:
49
+ ```bash
50
+ pushmi_pullyu start
51
+ ```
52
+
53
+ PushmiPullyu is quite flexible in that many options can be specified at the command line.
54
+ Use `pushmi_pullyu --help` to see all the command line options available.
55
+
56
+ Example:
57
+
58
+ ```
59
+ Usage: pushmi_pullyu [options] [start|stop|restart|run]
60
+
61
+ Specific options:
62
+ -a, --minimum-age AGE Minimum amount of time an item must spend in the queue, in seconds.
63
+ -d, --debug Enable debug logging
64
+ -r, --rollbar-token TOKEN Enable error reporting to Rollbar
65
+ -C, --config PATH Path for YAML config file
66
+ -L, --logdir PATH Path for directory to store log files
67
+ -D, --piddir PATH Path for directory to store pid files
68
+ -W, --workdir PATH Path for directory where AIP creation work takes place in
69
+ -N, --process_name NAME Name of the application process
70
+ -m, --monitor Start monitor process for a deamon
71
+ -q, --queue NAME Name of the queue to read from
72
+
73
+ Common options:
74
+ -v, --version Show version
75
+ -h, --help Show this message
76
+ ```
77
+
78
+ ## Configuration file
79
+
80
+ You can also provide a configuration file which PushmiPullyu will use with the -C (or --config) flag:
81
+
82
+ ```bash
83
+ pushmi_pullyu start -C /path/to/config.yml
84
+ ```
85
+
86
+ By default, if no configuration file is specified, PushmiPullyu will look for a configuration file at config/pushmi_pullyu.yml.
87
+
88
+ Take the following [sample configuration](https://github.com/ualbertalib/pushmi_pullyu/blob/master/examples/pushmi_pullyu.yml) as inspiration or check out the [DEFAULTS hash](https://github.com/ualbertalib/pushmi_pullyu/blob/master/lib/pushmi_pullyu.rb) in the main module to see all available options.
89
+
90
+ Options passed on the command line will also override options specified in the config file.
91
+
92
+ ## Signals
93
+
94
+ PushmiPullyu responds to these signals:
95
+
96
+ * `INT`: Equivalent of sending Ctrl-C to PushmiPullyu. Will attempt to finish then exit.
97
+ * `TERM`: Similar to `INT`. PushmiPullyu will attempt to finish then exit.
98
+ * `HUP`: Reopens log files as defined in logfile configuration options.
99
+
100
+ ## Development
101
+
102
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
103
+
104
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
105
+
106
+ ## Testing
107
+
108
+ To run the test suite:
109
+
110
+ ```bash
111
+ bundle install
112
+ bundle exec rake
113
+ ```
114
+
115
+ This will run both rspec and rubocop together.
116
+
117
+ To run rspec by itself:
118
+
119
+ ```bash
120
+ bundle exec rspec
121
+ ```
122
+ To run rubocop by itself:
123
+
124
+ ```bash
125
+ bundle exec rubocop
126
+ ```
127
+
128
+ ## Deployment
129
+
130
+ TODO: Fill me out
131
+
132
+ ## License
133
+
134
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
135
+
@@ -0,0 +1,11 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+ require 'rubocop/rake_task'
4
+
5
+ RSpec::Core::RakeTask.new do |r|
6
+ r.verbose = false
7
+ end
8
+
9
+ RuboCop::RakeTask.new
10
+
11
+ task default: [:spec, :rubocop]
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require 'pushmi_pullyu'
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require 'irb'
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
File without changes
@@ -0,0 +1,40 @@
1
+ # Sample configuration file for PushmiPullyu.
2
+ # Options here can still be overridden by cmd line args.
3
+
4
+ # Pass this file at config/pushmi_pullyu.yml and PushmiPullyu will
5
+ # pick it up automatically. Or pass this file into pushmi_pullyu on command line
6
+ # via `pushmi_pullyu -C path/to/pushmi_pullyu.yml`
7
+
8
+ # PushmiPullyu will run this file through ERB when reading it so you can
9
+ # even put in dynamic logic, like consuming ENV Variables.
10
+
11
+ debug: false
12
+ logdir: log
13
+ monitor: false
14
+ piddir: tmp/pids
15
+ workdir: tmp/work
16
+ process_name: pushmi_pullyu
17
+ queue_name: dev:pmpy_queue
18
+ minimum_age: 0
19
+ rollbar_token: 'abc123xyz'
20
+
21
+ redis:
22
+ host: localhost
23
+ port: 6379
24
+
25
+ solr:
26
+ url: http://localhost:8983/solr/development
27
+
28
+ fedora:
29
+ url: http://localhost:8983/fedora/rest
30
+ user: fedoraAdmin
31
+ password: fedoraAdmin
32
+ base_path: /dev
33
+
34
+ swift:
35
+ auth_version: v1.0
36
+ tenant: tester
37
+ username: test:tester
38
+ password: testing
39
+ endpoint: http://localhost:8080
40
+ container: ERA
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'pushmi_pullyu'
4
+
5
+ cli = PushmiPullyu::CLI.instance
6
+ cli.parse(ARGV)
7
+ cli.run
@@ -0,0 +1,95 @@
1
+ require 'pushmi_pullyu/logging'
2
+
3
+ require 'pushmi_pullyu/aip'
4
+ require 'pushmi_pullyu/aip/creator'
5
+ require 'pushmi_pullyu/aip/downloader'
6
+ require 'pushmi_pullyu/aip/solr_fetcher'
7
+ require 'pushmi_pullyu/aip/fedora_fetcher'
8
+ require 'pushmi_pullyu/cli'
9
+ require 'pushmi_pullyu/preservation_queue'
10
+ require 'pushmi_pullyu/swift_depositer'
11
+ require 'pushmi_pullyu/version'
12
+
13
+ require 'active_support'
14
+ require 'active_support/core_ext'
15
+
16
+ # PushmiPullyu main module
17
+ module PushmiPullyu
18
+ DEFAULTS = {
19
+ daemonize: false,
20
+ debug: false,
21
+ logdir: 'log',
22
+ minimum_age: 0,
23
+ monitor: false,
24
+ piddir: 'tmp/pids',
25
+ workdir: 'tmp/work',
26
+ process_name: 'pushmi_pullyu',
27
+ queue_name: 'dev:pmpy_queue',
28
+ redis: {
29
+ host: 'localhost',
30
+ port: 6379
31
+ },
32
+ # TODO: rest of these are examples for solr/fedora/swift... feel free to fill them in correctly
33
+ solr: {
34
+ url: 'http://localhost:8983/solr/development'
35
+ },
36
+ fedora: {
37
+ url: 'http://localhost:8983/fedora/rest',
38
+ user: 'fedoraAdmin',
39
+ password: 'fedoraAdmin',
40
+ base_path: '/dev'
41
+ },
42
+ swift: {
43
+ auth_version: 'v1.0',
44
+ tenant: 'tester',
45
+ username: 'test:tester',
46
+ password: 'testing',
47
+ endpoint: 'http://localhost:8080',
48
+ container: 'ERA'
49
+ }
50
+ }.freeze
51
+
52
+ def self.options
53
+ @options ||= DEFAULTS.dup
54
+ end
55
+
56
+ def self.options=(opts)
57
+ options.deep_merge!(opts)
58
+ end
59
+
60
+ def self.override_options(opts)
61
+ @options = opts
62
+ end
63
+
64
+ def self.application_log_file
65
+ "#{options[:logdir]}/#{options[:process_name]}.log"
66
+ end
67
+
68
+ def self.logger
69
+ PushmiPullyu::Logging.logger
70
+ end
71
+
72
+ def self.logger=(log)
73
+ PushmiPullyu::Logging.logger = log
74
+ end
75
+
76
+ def self.server_running=(status)
77
+ @server_running = status
78
+ end
79
+
80
+ def self.reset_logger=(status)
81
+ @reset_logger = status
82
+ end
83
+
84
+ def self.server_running?
85
+ @server_running
86
+ end
87
+
88
+ def self.continue_polling?
89
+ server_running? && !reset_logger?
90
+ end
91
+
92
+ def self.reset_logger?
93
+ @reset_logger
94
+ end
95
+ end