doi_extractor 1.0.5 → 1.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '092d2a2486613b1db9414d267cfccef66d4adc17'
-  data.tar.gz: 36df18eeaba0c2999ef7d7b448fa83543f1787c2
+  metadata.gz: 116f301792c96628d4f5517bd4f7efc864634341
+  data.tar.gz: d44bd7726b32645b97c1f6e929ad2e067ae028f2
 SHA512:
-  metadata.gz: 9022ebc38079cf1d779a006ca8a91aee2aab6f298245d2af300493a7653e4754be9e45df3d579e3374229c3991af24c9a41c7719eb451d9f10884cbcd5d8bdd1
-  data.tar.gz: b46e08cd9ef16a96c4d54e140b87e1a752200bc2cad89f1ca88af54eb3a9b02ce2ff2428a6404c1f61db118a1d18d7969882159cc1a33c7cafff75171cb1dfea
+  metadata.gz: 7fd0b1509417dd41a767c8b5ff8549f7232c06e7907af4531a8076401e0f75716f8706eb0372789412d9887d9932b9fc455f5f9589d0d03a636ddfcc8a14974f
+  data.tar.gz: f53dabd2f7125f14976657de55d977c4a6b5ab13f50dca19c1fa838bded20bc814d91a1dcbcdf9853ae53fd7304150ae0c76649576318b6e326e9e4053a72ac1

data/bin/doi_extractor

@@ -15,4 +15,4 @@ if parser.valid?
 else
   puts parser
   exit 1
-end
+end

data/config/deploy.rb

@@ -10,7 +10,7 @@ set :deploy_to, '/pkg/ipums/programming/doi_extractor'
 # Default value for :linked_files is []
 # append :linked_files, "config/database.yml", "config/secrets.yml"
 # Default value for linked_dirs is []
-# set :linked_dirs, %w{vendor/bundle}
+set :linked_dirs, %w{vendor/bundle log}
 
 set :app_version, DoiExtractor::VERSION
 set :jruby_path, -> { "/pkg/mpctools/jruby/current" }

data/doi_extractor.gemspec

@@ -17,9 +17,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'json'
+  spec.add_runtime_dependency 'jruby-openssl', '>= 0.9.18' if RUBY_PLATFORM =~ /java/
 
   spec.add_development_dependency "bundler", "> 1.10"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake", "< 12.3"
   spec.add_development_dependency "public_suffix", "1.4.6" # Locked to this version because of jruby in 1.9 mode
   spec.add_development_dependency "rspec", "~> 3.5.0"
   spec.add_development_dependency "ci_reporter_rspec", "~> 1.0.0"

data/lib/doi_extractor.rb

@@ -1,6 +1,7 @@
 require 'erb'
 require 'fileutils'
 require 'json'
+require 'logger'
 require 'net/https'
 require 'optparse'
 require 'optparse/uri'
@@ -20,6 +21,7 @@ require 'doi_extractor/errors'
 require 'doi_extractor/download_command'
 require 'doi_extractor/ipums_client'
 require 'doi_extractor/ipums_uri_builder'
+require 'doi_extractor/logger'
 require 'doi_extractor/options'
 require 'doi_extractor/secrets'
 require 'doi_extractor/status_command'

data/lib/doi_extractor/command.rb

@@ -17,26 +17,31 @@ module DoiExtractor
     end
 
     attr_reader :options, :start_time
-    attr_accessor :std_out, :user_input_callback
+    attr_accessor :std_out, :user_input_callback, :log
 
     def initialize(options, std_out = STDOUT, input_callback = nil)
       @options = options
       @std_out = std_out
       @user_input_callback = input_callback || default_input_callback
+      @log = Logger::get_log
     end
 
     def execute
       begin
         @start_time = Time.now
         cmd = self.class.name.split('::').last.sub('Command', '')
+        log.info "Starting #{cmd} Command:\n    #{options.to_hash}"
         say("starting #{cmd} Command", true)
         say("api: #{options.api_uri}", true)
         say("path: #{options.download_base_path}", true)
         _execute
+        log.info "Finished #{cmd} Command"
         say("Finished in #{(Time.now - @start_time).round(2)} seconds", true)
       rescue CommandFailError => e
+        log.error "#{e.class}: #{e.message}\n    #{e.backtrace.join("\n    ")}"
         raise e
       rescue => e
+        log.error "#{e.class}: #{e.message}\n    #{e.backtrace.join("\n    ")}"
         raise CommandFailError.new('An Unexpected Error Has Occurred', e)
       end
     end

data/lib/doi_extractor/download_location.rb

@@ -79,7 +79,7 @@ module DoiExtractor
 
     def extract_request_name(er)
       s = er.samples.first
-      [s.name, s.long_description].compact.join(' - ')
+      [s.name, s.long_description].compact.join(' - ').gsub('/', '_')
     end
 
   end

data/lib/doi_extractor/ipums_client.rb

@@ -15,6 +15,8 @@ module DoiExtractor
           raise 'Invalid uri'
       end
 
+      raise("Invalid uri: #{uri}") unless @uri.is_a?(URI::HTTP)
+
       @username = username
       @password = password
     end

data/lib/doi_extractor/ipums_uri_builder.rb

@@ -11,11 +11,17 @@ module DoiExtractor
         highered: IpumsConfig.new('highered.ipums.org'),
         nhis:     IpumsConfig.new('nhis.ipums.org'),
         ipumsi:   IpumsConfig.new('international.ipums.org', nil, 'international'),
+        meps:     IpumsConfig.new('meps.ipums.org'),
         mtus:     IpumsConfig.new('mtusdata.org', 'www.'),
         napp:     IpumsConfig.new('nappdata.org', 'www.'),
+        pma:      IpumsConfig.new('pma.ipums.org'),
         usa:      IpumsConfig.new('usa.ipums.org'),
     }
 
+    def self.project_list
+      CONFIGS.keys.map(&:to_s)
+    end
+
     attr_reader :project, :environment, :config
 
     def initialize(project, environment)

data/lib/doi_extractor/logger.rb

@@ -0,0 +1,16 @@
+module DoiExtractor
+  module Logger
+    def self.get_log
+      pkg_path = '/pkg/ipums/programming/doi_extractor/current/log'
+      local_path = File.expand_path(File.dirname(__FILE__) + '/../../log')
+
+      if Dir::exists?(pkg_path) && File.writable?(pkg_path)
+        log_path = pkg_path
+      else
+        log_path = local_path
+      end
+
+      ::Logger.new("#{log_path}/doi_extractor.log", 15, 1073741824)
+    end
+  end
+end

data/lib/doi_extractor/options.rb

@@ -1,7 +1,7 @@
 module DoiExtractor
   class Options
 
-    PROJECTS = %w(ahtus atus cps dhs highered nhis ipumsi mtus napp usa)
+    PROJECTS = IpumsUriBuilder::project_list
     ENVIRONMENTS = %w(local demo internal staging live)
     COMMANDS = %w(create download status cancel)
     
@@ -101,6 +101,10 @@ module DoiExtractor
       errors.empty?
     end
 
+    def to_hash
+      Hash[OPTIONS.map { |o| [o, self.send(o)] }].merge(errors: self.errors)
+    end
+
     private
 
     def calculate_api_uri
@@ -113,8 +117,11 @@ module DoiExtractor
 
     def calculate_download_base_path
       if project && environment
-        # "/pkg/ipums/programming/#{project}/#{environment}"
-        "/pkg/ipums/programming/doi_extractor/temp_archive/#{project}/#{environment}"
+        if environment == 'live'
+          "/pkg/ipums/preservationarchive/#{project}"
+        else
+          "/pkg/ipums/programming/doi_extractor/temp_archive/#{project}/#{environment}"
+        end
       else
         nil
       end

data/lib/doi_extractor/status_command.rb

@@ -50,6 +50,7 @@ module DoiExtractor
 
       say("DOI Extract (ID: #{doi_extract.id})")
       say("\tStatus:         #{doi_extract.status}")
+      say("\tCreated:        #{doi_extract.created_at}")
       say("\tSubmitted:      #{doi_extract.submit_date}")
       say("\tFinished:       #{doi_extract.finish_date}")
       say("\tData Available: #{doi_extract.all_data_available}")

data/lib/doi_extractor/version.rb

@@ -1,3 +1,3 @@
 module DoiExtractor
-  VERSION = '1.0.5'
+  VERSION = '1.0.16'
 end

data/spec/unit/download_location_spec.rb

@@ -32,6 +32,11 @@ describe DoiExtractor::DownloadLocation do
     expect(location.extract_request_path(extract)).to eq File.join(path, '1990a - US 1990 5%.tar.gz')
   end
 
+  it 'handles slashes in sample names' do
+    extract.samples.first.long_description = 'US 1990 Urban/Rural'
+    expect(location.extract_request_path(extract)).to eq File.join(path, '1990a - US 1990 Urban_Rural.tar.gz')
+  end
+
   it 'can create itself' do
     expect(Dir.exist?(path)).to be_falsey
     location.ensure!

data/spec/unit/ipums_client_spec.rb

@@ -13,9 +13,9 @@ describe DoiExtractor::IpumsClient do
   end
 
   it 'handles HTTP 500' do
-    stub_request(:get, 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1').to_return(status: 500, body: '{"error": "bad stuff"}')
+    stub_request(:get, 'https://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1').to_return(status: 500, body: '{"error": "bad stuff"}')
 
-    client = DoiExtractor::IpumsClient.new('http://localhost:3003/cps-action', 'user', 'pass')
+    client = DoiExtractor::IpumsClient.new('https://localhost:3003/cps-action', 'user', 'pass')
 
     expect { client.get_doi_extract(1) }.to raise_error(DoiExtractor::ApiError, /invalid/i)
   end

data/spec/unit/options_spec.rb

@@ -57,11 +57,17 @@ describe DoiExtractor::Options do
   end
 
   context 'when setting path values' do
-    let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'live', extract_group_id: 1) }
+    let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'internal', extract_group_id: 1) }
 
     it 'defaults the base path' do
       expect(options.valid?).to be_truthy
-      expect(options.download_base_path).to eq "/pkg/ipums/programming/doi_extractor/temp_archive/cps/live"
+      expect(options.download_base_path).to eq "/pkg/ipums/programming/doi_extractor/temp_archive/cps/internal"
+    end
+
+    it 'defaults the base path for live' do
+      options.environment = 'live'
+      expect(options.valid?).to be_truthy
+      expect(options.download_base_path).to eq "/pkg/ipums/preservationarchive/cps"
     end
 
     it 'allows overriding the base path' do
@@ -83,4 +89,11 @@ describe DoiExtractor::Options do
     expect(opts.api_password).to eq '1234'
   end
 
+  it 'supports meps' do
+    opts = DoiExtractor::Options.for_command('download', project: 'meps', environment: 'internal', extract_group_id: 1)
+    opts.validate
+    expect(opts.valid?).to eq true
+    expect(opts.api_uri).to eq 'https://internal.meps.ipums.org/meps-action'
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: doi_extractor
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.16
 platform: ruby
 authors:
 - Dan Elbert
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-19 00:00:00.000000000 Z
+date: 2018-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -42,16 +42,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -163,6 +163,7 @@ files:
 - lib/doi_extractor/errors.rb
 - lib/doi_extractor/ipums_client.rb
 - lib/doi_extractor/ipums_uri_builder.rb
+- lib/doi_extractor/logger.rb
 - lib/doi_extractor/old_ruby_patch.rb
 - lib/doi_extractor/options.rb
 - lib/doi_extractor/secrets.rb
@@ -231,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Command line tool to package IPUMS DOI extracts