pushmi_pullyu 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 392efadaf852092c534f920017037957bb22436c
+  data.tar.gz: e297ea545d6c31ad594721d9c952e9e6a0ccf719
+SHA512:
+  metadata.gz: 751a8b7ce779203c369a41680cb45b394cb67062ff6fdf9c1e944015633ff3feeef50992a8a0b6ee7cc764c270bacad76d407c5b5c9737479417d64b7c6dab57
+  data.tar.gz: 1e2343f0960560cdfc512ddc5a447b1dd80cafd6efae98b16bb39937d01a4798a4f49eb33ce9a7174c23f8d83d8af1578c9b2f889d29593265cff8bff891f56c

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.editorconfig

@@ -0,0 +1,16 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{diff,md}]
+trim_trailing_whitespace = false

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/log/
+/tmp/
+/vendor/bundle
+*.gem
+
+# rspec failure tracking
+.rspec_status

data/.hound.yml

@@ -0,0 +1,2 @@
+ruby:
+  config_file: .rubocop.yml

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,90 @@
+# List of cops can be found here:
+# https://github.com/bbatsov/rubocop/blob/master/config/default.yml
+# https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
+# https://github.com/bbatsov/rubocop/blob/master/config/enabled.yml
+require: rubocop-rspec
+
+AllCops:
+  DisplayCopNames: true
+  DisplayStyleGuide: true
+  Exclude:
+    - 'tmp/**/*'
+    - 'vendor/**/*'
+  ExtraDetails: true
+  TargetRubyVersion: 2.3
+
+# readability is Actually Good
+Layout/EmptyLinesAroundClassBody:
+  EnforcedStyle: empty_lines_except_namespace
+
+Layout/IndentationConsistency:
+  Enabled: true
+  EnforcedStyle: normal
+
+# A calculated magnitude based on number of assignments,
+# branches, and conditions.
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 500 # default 100
+
+# A complexity metric that is strongly correlated to the number
+# of test cases needed to validate a method.
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: true
+  Max: 120 # default is 80
+
+# Avoid methods longer than 10 lines of code.
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Max: 500 # default 100
+
+# A complexity metric geared towards measuring complexity for a human reader.
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+# indentation is an endangered resource
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+
+Style/Documentation:
+  Enabled: false
+
+Style/FileName:
+  Exclude:
+    - Dangerfile
+    - Rakefile
+    - Gemfile
+
+# Checks if there is a magic comment to enforce string literals
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# Perfer to use // but %r can be better in certain scenarios
+Style/RegexpLiteral:
+  Enabled: false
+
+Style/SymbolArray:
+  Enabled: false
+
+# Use %w or %W for arrays of words.
+Style/WordArray:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/DescribedClass:
+  EnforcedStyle: explicit

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+sudo: false
+cache: bundler
+services:
+  - redis-server
+
+before_install:
+  - gem install bundler
+  - gem update bundler
+
+rvm:
+  - 2.3.3
+  - 2.4.0
+
+before_script:
+  - bundle exec danger

data/Dangerfile

@@ -0,0 +1,22 @@
+# Make sure non-trivial amounts of code changes come with corresponding tests
+has_app_changes = !git.modified_files.grep(/lib/).empty?
+has_spec_changes = !git.modified_files.grep(/spec/).empty?
+
+if git.lines_of_code > 50 && has_app_changes && !has_spec_changes
+  warn('There are code changes, but no corresponding tests. '\
+         'Please include tests if this PR introduces any modifications in '\
+         'behavior.',
+       sticky: false)
+end
+
+# Mainly to encourage writing up some reasoning about the PR, rather than
+# just leaving a title
+warn('Please add a detailed summary in the description.') if github.pr_body.length < 5
+
+# Ensure a clean commits history
+if git.commits.any? { |c| c.message =~ /^Merge branch '#{github.branch_for_base}'/ }
+  warn 'Please rebase to get rid of the merge commits in this PR'
+end
+
+# Warn when there is a big PR
+warn('This PR is too big! Consider breaking it down into smaller PRs.') if git.lines_of_code > 500

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pushmi_pullyu.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 University of Alberta Libraries
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,135 @@
+<p align="center">
+  <a href="https://github.com/ualbertalib/pushmi_pullyu/">
+    <img src="docs/images/pushmi-pullyu.png" alt="PushmiPullyu logo">
+  </a>
+</p>
+
+[![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
+[![Dependency Status](https://gemnasium.com/badges/github.com/ualbertalib/pushmi_pullyu.svg)](https://gemnasium.com/github.com/ualbertalib/pushmi_pullyu)
+[![Build Status](https://travis-ci.org/ualbertalib/pushmi_pullyu.svg?branch=master)](https://travis-ci.org/ualbertalib/pushmi_pullyu)
+[![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
+
+PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
+
+Its primary job is to manage the flow of content from Fedora into Swift for preservation.
+
+![System Infrastructure Diagram](docs/images/system-infrastructure-diagram.png)
+
+## Workflow
+
+1.  Any save (create or update) on a GenericFile in ERA will trigger an after save callback that will push the GenericFile unique identifier (NOID) into a Queue.
+2. The queue (Redis) is setup to be a unique set (which only allows one GenericFile NOID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
+3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
+4. All the GenericFile information and data required for preservation are retrieved from Fedora and Solr using multiple REST calls.
+5. An Archival Information Package (AIP) is created from the GenericFile's information. It is then bagged and tarred.
+6. The AIP tar is then uploaded to Swift via a REST call.
+7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
+
+
+## Requirements
+
+PushmiPullyu supports Ruby 2.3.1+
+
+## Installation
+
+PushmiPullyu is hosted on rubygems.org. Therefore it can be installed via:
+
+```bash
+  gem install pushmi_pullyu
+```
+
+## Usage
+
+Foreground mode:
+```bash
+  pushmi_pullyu
+```
+
+Run as a daemon:
+```bash
+  pushmi_pullyu start
+```
+
+PushmiPullyu is quite flexible in that many options can be specified at the command line.
+Use `pushmi_pullyu --help` to see all the command line options available.
+
+Example:
+
+```
+Usage: pushmi_pullyu [options] [start|stop|restart|run]
+
+Specific options:
+  -a, --minimum-age AGE            Minimum amount of time an item must spend in the queue, in seconds.
+  -d, --debug                      Enable debug logging
+  -r, --rollbar-token TOKEN        Enable error reporting to Rollbar
+  -C, --config PATH                Path for YAML config file
+  -L, --logdir PATH                Path for directory to store log files
+  -D, --piddir PATH                Path for directory to store pid files
+  -W, --workdir PATH               Path for directory where AIP creation work takes place in
+  -N, --process_name NAME          Name of the application process
+  -m, --monitor                    Start monitor process for a deamon
+  -q, --queue NAME                 Name of the queue to read from
+
+Common options:
+  -v, --version                    Show version
+  -h, --help                       Show this message
+```
+
+## Configuration file
+
+You can also provide a configuration file which PushmiPullyu will use with the -C (or --config) flag:
+
+```bash
+  pushmi_pullyu start -C /path/to/config.yml
+```
+
+By default, if no configuration file is specified, PushmiPullyu will look for a configuration file at config/pushmi_pullyu.yml.
+
+Take the following [sample configuration](https://github.com/ualbertalib/pushmi_pullyu/blob/master/examples/pushmi_pullyu.yml) as inspiration or check out the [DEFAULTS hash](https://github.com/ualbertalib/pushmi_pullyu/blob/master/lib/pushmi_pullyu.rb) in the main module to see all available options.
+
+Options passed on the command line will also override options specified in the config file.
+
+## Signals
+
+PushmiPullyu responds to these signals:
+
+* `INT`: Equivalent of sending Ctrl-C to PushmiPullyu. Will attempt to finish then exit.
+* `TERM`: Similar to `INT`. PushmiPullyu will attempt to finish then exit.
+* `HUP`: Reopens log files as defined in logfile configuration options.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Testing
+
+To run the test suite:
+
+```bash
+  bundle install
+  bundle exec rake
+```
+
+This will run both rspec and rubocop together.
+
+To run rspec by itself:
+
+```bash
+  bundle exec rspec
+```
+To run rubocop by itself:
+
+```bash
+  bundle exec rubocop
+```
+
+## Deployment
+
+TODO: Fill me out
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new do |r|
+  r.verbose = false
+end
+
+RuboCop::RakeTask.new
+
+task default: [:spec, :rubocop]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'pushmi_pullyu'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/pushmi_pullyu.yml

@@ -0,0 +1,40 @@
+# Sample configuration file for PushmiPullyu.
+# Options here can still be overridden by cmd line args.
+
+# Pass this file at config/pushmi_pullyu.yml and PushmiPullyu will
+# pick it up automatically. Or pass this file into pushmi_pullyu on command line
+# via `pushmi_pullyu -C path/to/pushmi_pullyu.yml`
+
+# PushmiPullyu will run this file through ERB when reading it so you can
+# even put in dynamic logic, like consuming ENV Variables.
+
+debug: false
+logdir: log
+monitor: false
+piddir: tmp/pids
+workdir: tmp/work
+process_name: pushmi_pullyu
+queue_name: dev:pmpy_queue
+minimum_age: 0
+rollbar_token: 'abc123xyz'
+
+redis:
+  host: localhost
+  port: 6379
+
+solr:
+  url: http://localhost:8983/solr/development
+
+fedora:
+  url: http://localhost:8983/fedora/rest
+  user: fedoraAdmin
+  password: fedoraAdmin
+  base_path: /dev
+
+swift:
+  auth_version: v1.0
+  tenant: tester
+  username: test:tester
+  password: testing
+  endpoint: http://localhost:8080
+  container: ERA

data/exe/pushmi_pullyu

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'pushmi_pullyu'
+
+cli = PushmiPullyu::CLI.instance
+cli.parse(ARGV)
+cli.run

data/lib/pushmi_pullyu.rb

@@ -0,0 +1,95 @@
+require 'pushmi_pullyu/logging'
+
+require 'pushmi_pullyu/aip'
+require 'pushmi_pullyu/aip/creator'
+require 'pushmi_pullyu/aip/downloader'
+require 'pushmi_pullyu/aip/solr_fetcher'
+require 'pushmi_pullyu/aip/fedora_fetcher'
+require 'pushmi_pullyu/cli'
+require 'pushmi_pullyu/preservation_queue'
+require 'pushmi_pullyu/swift_depositer'
+require 'pushmi_pullyu/version'
+
+require 'active_support'
+require 'active_support/core_ext'
+
+# PushmiPullyu main module
+module PushmiPullyu
+  DEFAULTS = {
+    daemonize: false,
+    debug: false,
+    logdir: 'log',
+    minimum_age: 0,
+    monitor: false,
+    piddir: 'tmp/pids',
+    workdir: 'tmp/work',
+    process_name: 'pushmi_pullyu',
+    queue_name: 'dev:pmpy_queue',
+    redis: {
+      host: 'localhost',
+      port: 6379
+    },
+    # TODO: rest of these are examples for solr/fedora/swift... feel free to fill them in correctly
+    solr: {
+      url: 'http://localhost:8983/solr/development'
+    },
+    fedora: {
+      url: 'http://localhost:8983/fedora/rest',
+      user: 'fedoraAdmin',
+      password: 'fedoraAdmin',
+      base_path: '/dev'
+    },
+    swift: {
+      auth_version: 'v1.0',
+      tenant: 'tester',
+      username: 'test:tester',
+      password: 'testing',
+      endpoint: 'http://localhost:8080',
+      container: 'ERA'
+    }
+  }.freeze
+
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+
+  def self.options=(opts)
+    options.deep_merge!(opts)
+  end
+
+  def self.override_options(opts)
+    @options = opts
+  end
+
+  def self.application_log_file
+    "#{options[:logdir]}/#{options[:process_name]}.log"
+  end
+
+  def self.logger
+    PushmiPullyu::Logging.logger
+  end
+
+  def self.logger=(log)
+    PushmiPullyu::Logging.logger = log
+  end
+
+  def self.server_running=(status)
+    @server_running = status
+  end
+
+  def self.reset_logger=(status)
+    @reset_logger = status
+  end
+
+  def self.server_running?
+    @server_running
+  end
+
+  def self.continue_polling?
+    server_running? && !reset_logger?
+  end
+
+  def self.reset_logger?
+    @reset_logger
+  end
+end