doi_extractor 1.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: '092d2a2486613b1db9414d267cfccef66d4adc17'
+  data.tar.gz: 36df18eeaba0c2999ef7d7b448fa83543f1787c2
+SHA512:
+  metadata.gz: 9022ebc38079cf1d779a006ca8a91aee2aab6f298245d2af300493a7653e4754be9e45df3d579e3374229c3991af24c9a41c7719eb451d9f10884cbcd5d8bdd1
+  data.tar.gz: b46e08cd9ef16a96c4d54e140b87e1a752200bc2cad89f1ca88af54eb3a9b02ce2ff2428a6404c1f61db118a1d18d7969882159cc1a33c7cafff75171cb1dfea

data/.gitignore

@@ -0,0 +1,54 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/spec/testdata/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+/log/
+
+Gemfile.lock
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Capfile

@@ -0,0 +1,34 @@
+# Load DSL and set up stages
+require "capistrano/setup"
+
+# Include default deployment tasks
+require "capistrano/deploy"
+
+# Include doi_extractor version
+require File.expand_path('../lib/doi_extractor/version', __FILE__)
+
+# Load the SCM plugin appropriate to your project:
+require "capistrano/scm/git"
+install_plugin Capistrano::SCM::Git
+
+# Include tasks from other gems included in your Gemfile
+#
+# For documentation on these, see for example:
+#
+#   https://github.com/capistrano/rvm
+#   https://github.com/capistrano/rbenv
+#   https://github.com/capistrano/chruby
+#   https://github.com/capistrano/bundler
+#   https://github.com/capistrano/rails
+#   https://github.com/capistrano/passenger
+#
+# require "capistrano/rvm"
+# require "capistrano/rbenv"
+# require "capistrano/chruby"
+require "capistrano/bundler"
+# require "capistrano/rails/assets"
+# require "capistrano/rails/migrations"
+# require "capistrano/passenger"
+
+# Load custom tasks from `lib/capistrano/tasks` if you have any defined
+Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r }

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem "capistrano", "3.8.1", platform: :ruby
+gem 'capistrano-bundler', '~> 1.2', platform: :ruby

data/README.md

@@ -0,0 +1,50 @@
+# DOI Extractor
+
+## Use
+
+The DOI Extractor is a command line tool for generating, submitting, downloading,
+and packaging a set of extracts for DOI archival.  It currently supports 4 commands:
+`cancel`, `create`, `download`, and `status`.
+
+#### Cancel
+Cancels a submitted DOI extract group.  Will fail all associated pending extracts.
+
+#### Create
+Creates a new DOI extract group and its associated extracts, then submits them.
+Optionally takes an email address and will send a notification when all extracts are
+ready to download.
+
+#### Download
+Once all extracts for a DOI group are complete, this command will download and package them.
+
+#### Status
+Prints summary information about recent DOI extracts or detailed information about a
+particular extract
+
+### Further Information
+
+See `doi_extractor <COMMAND> --help` for more detailed options.
+
+## Examples
+
+### Create and submit a new DOI extract group for USA
+`doi_extractor create -p usa -v 6.5 -E delbert@umn.edu`
+
+### Check the status of a submitted extract group
+`doi_extractor status -p usa -i 5`
+
+### Download the completed extracts to the default location
+`doi_extractor download -p usa -i 5`
+
+## Deployment
+
+The DOI Extractor is a gem that gets installed to the shared jruby environment
+(/pkg/mpctools/jruby).
+
+It may be deployed using capistrano: `cap production deploy`
+
+### Ruby Version Notes
+
+This gem uses capistrano 3.8, which is not supported under jruby 1.7.  For deployment, use a modern ruby (MRI works fine).
+
+The CI build uses jruby to ensure the Gemfile is valid in that env and that all specs pass.

data/Rakefile

@@ -0,0 +1,43 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+require 'ci/reporter/rake/rspec'
+
+RSpec::Core::RakeTask.new(:spec => 'ci:setup:rspec')
+
+task :test => :spec
+task :default => :spec
+
+task :disable_rubygems do
+  ENV['gem_push'] = 'no'
+end
+
+Rake::Task[:release].enhance [:disable_rubygems]
+
+namespace :spec do
+
+  desc "integration test the JSON API endpoints"
+  RSpec::Core::RakeTask.new(:integration) do |t|
+    # set the RAILS_ENV such that :integration tagged
+    # specs are run
+    ENV['ENABLE_INTEGRATION'] = 'true'
+
+    # only run those files in the 'integration' directory
+    t.pattern = "./spec/integration{,/*/**}/*_spec.rb"
+  end
+
+  desc 'Run unit and integration tests'
+  task :all do
+    ENV['ENABLE_INTEGRATION'] = 'true'
+    Rake::Task[:spec].invoke
+  end
+end
+
+namespace :test do
+  task :integration => 'spec:integration'
+  task :all => 'spec:all'
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -I lib -r doi_extractor.rb"
+end

data/bin/doi_extractor

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'doi_extractor'
+
+parser = DoiExtractor::CommandLineParser.parse(ARGV)
+
+if parser.valid?
+  cmd = DoiExtractor::Command.for(parser.options)
+  begin
+    cmd.execute
+  rescue DoiExtractor::CommandFailError => e
+    STDERR.puts e.error_report
+    exit e.exit_code
+  end
+else
+  puts parser
+  exit 1
+end

data/config/deploy.rb

@@ -0,0 +1,34 @@
+# config valid only for current version of Capistrano
+lock "3.8.1"
+
+set :application, 'doi_extractor'
+set :repo_url, 'git@github.umn.edu:mpc/doi_extractor.git'
+set :branch, `git rev-parse --abbrev-ref HEAD`.chomp
+set :deploy_to, '/pkg/ipums/programming/doi_extractor'
+
+
+# Default value for :linked_files is []
+# append :linked_files, "config/database.yml", "config/secrets.yml"
+# Default value for linked_dirs is []
+# set :linked_dirs, %w{vendor/bundle}
+
+set :app_version, DoiExtractor::VERSION
+set :jruby_path, -> { "/pkg/mpctools/jruby/current" }
+set :default_env, -> { {path: "#{fetch(:jruby_path)}/bin:$PATH"} }
+set :keep_releases, 5
+set :user, `logname`.chomp
+
+set :bundle_flags, '--quiet'
+set :bundle_without, 'deployment'
+
+after 'deploy:published', :jruby_install
+
+desc 'Installs the gem into the shared jruby env'
+task :jruby_install do
+  on roles(:app) do
+    within current_path do
+      execute :rake, 'build'
+      execute "PATH=#{fetch(:jruby_path)}/bin:$PATH gem install --wrapper --bindir #{fetch(:jruby_path)}/bin #{current_path}/pkg/doi_extractor-#{fetch(:app_version)}.gem"
+    end
+  end
+end

data/config/deploy/production.rb

@@ -0,0 +1,65 @@
+
+server 'ipums-internal-web.pop.umn.edu', user: fetch(:user), roles: %w(app)
+
+
+# server-based syntax
+# ======================
+# Defines a single server with a list of roles and multiple properties.
+# You can define all roles on a single server, or split them:
+
+# server "example.com", user: "deploy", roles: %w{app db web}, my_property: :my_value
+# server "example.com", user: "deploy", roles: %w{app web}, other_property: :other_value
+# server "db.example.com", user: "deploy", roles: %w{db}
+
+
+
+# role-based syntax
+# ==================
+
+# Defines a role with one or multiple servers. The primary server in each
+# group is considered to be the first unless any hosts have the primary
+# property set. Specify the username and a domain or IP for the server.
+# Don't use `:all`, it's a meta role.
+
+# role :app, %w{deploy@example.com}, my_property: :my_value
+# role :web, %w{user1@primary.com user2@additional.com}, other_property: :other_value
+# role :db,  %w{deploy@example.com}
+
+
+
+# Configuration
+# =============
+# You can set any configuration variable like in config/deploy.rb
+# These variables are then only loaded and set in this stage.
+# For available Capistrano configuration variables see the documentation page.
+# http://capistranorb.com/documentation/getting-started/configuration/
+# Feel free to add new variables to customise your setup.
+
+
+
+# Custom SSH Options
+# ==================
+# You may pass any option but keep in mind that net/ssh understands a
+# limited set of options, consult the Net::SSH documentation.
+# http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
+#
+# Global options
+# --------------
+#  set :ssh_options, {
+#    keys: %w(/home/rlisowski/.ssh/id_rsa),
+#    forward_agent: false,
+#    auth_methods: %w(password)
+#  }
+#
+# The server-based syntax can be used to override options:
+# ------------------------------------
+# server "example.com",
+#   user: "user_name",
+#   roles: %w{web app},
+#   ssh_options: {
+#     user: "user_name", # overrides user setting above
+#     keys: %w(/home/user_name/.ssh/id_rsa),
+#     forward_agent: false,
+#     auth_methods: %w(publickey password)
+#     # password: "please use keys"
+#   }

data/doi_extractor.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'doi_extractor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "doi_extractor"
+  spec.version       = DoiExtractor::VERSION
+  spec.authors       = ["Dan Elbert"]
+  spec.email         = ["delbert@umn.edu"]
+  spec.homepage      = 'https://github.umn.edu/mpc/doi_extractor'
+  spec.summary       = %q{Command line tool to package IPUMS DOI extracts}
+
+  spec.files         = Dir['bin/**/*'] + Dir['config/**/*'] + Dir['lib/**/*'] + Dir['spec/**/*'] + %w(.gitignore Capfile doi_extractor.gemspec Gemfile Rakefile README.md)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'json'
+
+  spec.add_development_dependency "bundler", "> 1.10"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "public_suffix", "1.4.6" # Locked to this version because of jruby in 1.9 mode
+  spec.add_development_dependency "rspec", "~> 3.5.0"
+  spec.add_development_dependency "ci_reporter_rspec", "~> 1.0.0"
+  spec.add_development_dependency "simplecov", "~> 0.13.0"
+  spec.add_development_dependency "simplecov-rcov", "~> 0.2.3"
+  spec.add_development_dependency "webmock", "~> 2.3.2"
+end

data/lib/doi_extractor.rb

@@ -0,0 +1,25 @@
+require 'erb'
+require 'fileutils'
+require 'json'
+require 'net/https'
+require 'optparse'
+require 'optparse/uri'
+require 'ostruct'
+require 'stringio'
+require 'uri'
+require 'yaml'
+
+require 'doi_extractor/old_ruby_patch'
+require 'doi_extractor/version'
+require 'doi_extractor/command'
+require 'doi_extractor/cancel_command'
+require 'doi_extractor/command_line_parser'
+require 'doi_extractor/create_command'
+require 'doi_extractor/download_location'
+require 'doi_extractor/errors'
+require 'doi_extractor/download_command'
+require 'doi_extractor/ipums_client'
+require 'doi_extractor/ipums_uri_builder'
+require 'doi_extractor/options'
+require 'doi_extractor/secrets'
+require 'doi_extractor/status_command'

data/lib/doi_extractor/cancel_command.rb

@@ -0,0 +1,24 @@
+module DoiExtractor
+  class CancelCommand < Command
+    def _execute
+      doi = ipums_client.get_doi_extract(options.extract_group_id)
+      unless doi
+        fail('No extract group found')
+      end
+
+      if options.force
+        say('skipping user confirmation', true)
+      else
+        i = ask("Are you sure you want to cancel DOI extract with ID #{doi.id}?")
+        if i.to_s.downcase[0...1] != 'y'
+          fail('user cancelled')
+        end
+      end
+
+      ipums_client.cancel_doi_extract(doi.id)
+
+      say("Complete")
+
+    end
+  end
+end

data/lib/doi_extractor/command.rb

@@ -0,0 +1,81 @@
+module DoiExtractor
+  class Command
+
+    def self.for(options)
+      case options.command
+        when 'status'
+          StatusCommand.new(options)
+        when 'create'
+          CreateCommand.new(options)
+        when 'download'
+          DownloadCommand.new(options)
+        when 'cancel'
+          CancelCommand.new(options)
+        else
+          raise "Invalid options: #{options.inspect}"
+      end
+    end
+
+    attr_reader :options, :start_time
+    attr_accessor :std_out, :user_input_callback
+
+    def initialize(options, std_out = STDOUT, input_callback = nil)
+      @options = options
+      @std_out = std_out
+      @user_input_callback = input_callback || default_input_callback
+    end
+
+    def execute
+      begin
+        @start_time = Time.now
+        cmd = self.class.name.split('::').last.sub('Command', '')
+        say("starting #{cmd} Command", true)
+        say("api: #{options.api_uri}", true)
+        say("path: #{options.download_base_path}", true)
+        _execute
+        say("Finished in #{(Time.now - @start_time).round(2)} seconds", true)
+      rescue CommandFailError => e
+        raise e
+      rescue => e
+        raise CommandFailError.new('An Unexpected Error Has Occurred', e)
+      end
+    end
+
+    protected
+
+    # Subclasses should override this method
+    def _execute
+    end
+
+    def default_input_callback
+      ->(msg) { say_nb("#{msg}\n(y/n) "); STDIN.gets }
+    end
+
+    def ask(msg)
+      user_input_callback.call(msg)
+    end
+
+    def say_nb(msg, is_verbose = false)
+      if options.verbose || !is_verbose
+        std_out.print msg
+      end
+    end
+
+    def say(msg, is_verbose = false)
+      say_nb(msg + "\n", is_verbose)
+    end
+
+    def fail(msg, ex = nil, code = 1)
+      raise CommandFailError.new(msg, ex, code)
+    end
+
+    def ipums_client
+      @ipums_client ||= IpumsClient.new(options.api_uri, secrets.api_username, secrets.api_password)
+    end
+
+    def secrets
+      @secrets ||= Secrets.new({api_username: options.api_username, api_password: options.api_password})
+    end
+
+  end
+end