doi_extractor 1.0.5

data/spec/reports/SPEC-DoiExtractor-DownloadCommand.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::DownloadCommand" tests="0" time="0.000123" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-DownloadLocation.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::DownloadLocation" tests="4" time="0.021346" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::DownloadLocation calculates extract request paths" time="0.000391">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadLocation can create itself" time="0.000702">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadLocation creates and clears the tmp path" time="0.000797">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadLocation packages up downloaded extract files" time="0.018921">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-IpumsClient.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::IpumsClient" tests="2" time="0.005526" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::IpumsClient handles HTTP 422" time="0.002534">
+  </testcase>
+  <testcase name="DoiExtractor::IpumsClient handles HTTP 500" time="0.002482">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-internal-environment.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::IpumsUriBuilder internal environment" tests="1" time="0.000353" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::IpumsUriBuilder internal environment builds correct urls" time="0.000117">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-live-environment.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::IpumsUriBuilder live environment" tests="1" time="0.000317" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::IpumsUriBuilder live environment builds correct urls" time="0.000186">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-local-environment.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::IpumsUriBuilder local environment" tests="1" time="0.000334" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::IpumsUriBuilder local environment builds correct urls" time="0.000127">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-IpumsUriBuilder.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::IpumsUriBuilder" tests="0" time="6.6e-05" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command-with-invalid-doi-version.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options.for_command with create command with invalid doi version" tests="1" time="0.000343" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options.for_command with create command with invalid doi version is not valid" time="0.000167">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options.for_command with create command" tests="2" time="0.000517" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options.for_command with create command returns the right class" time="0.000158">
+  </testcase>
+  <testcase name="DoiExtractor::Options.for_command with create command sets the attributes" time="0.000162">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-for-command-with-download-command.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options.for_command with download command" tests="2" time="0.000534" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options.for_command with download command returns the right class" time="0.00013">
+  </testcase>
+  <testcase name="DoiExtractor::Options.for_command with download command sets the attributes" time="0.000134">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-for-command-with-status-command.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options.for_command with status command" tests="2" time="0.000515" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options.for_command with status command returns the right class" time="0.000128">
+  </testcase>
+  <testcase name="DoiExtractor::Options.for_command with status command sets the attributes" time="0.000122">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-for-command.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options.for_command" tests="0" time="5.0e-05" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options-when-setting-path-values.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options when setting path values" tests="3" time="0.000766" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options when setting path values defaults the base path" time="0.000145">
+  </testcase>
+  <testcase name="DoiExtractor::Options when setting path values allows overriding the base path" time="0.000137">
+  </testcase>
+  <testcase name="DoiExtractor::Options when setting path values does not allow an empty base path" time="0.000143">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Options.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Options" tests="1" time="0.000294" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Options defaults to basic auth creds on internal env" time="0.000171">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-Secrets.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::Secrets" tests="2" time="0.003972" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::Secrets reads the api password" time="0.003576">
+  </testcase>
+  <testcase name="DoiExtractor::Secrets overrides correctly" time="0.000128">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-StatusCommand.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::StatusCommand" tests="1" time="0.004027" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::StatusCommand generates some output" time="0.003642">
+  </testcase>
+</testsuite>

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'simplecov'
+require 'simplecov-rcov'
+
+SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+SimpleCov.start do
+  add_filter '/spec/'
+  add_filter '/vendor/bundle'
+end
+
+require 'rspec'
+require 'webmock/rspec'
+require File.expand_path('../../lib/doi_extractor', __FILE__)
+
+Dir[File.expand_path('../support/**/*', __FILE__)].each { |f| require f }
+
+RSpec.configure do |config|
+
+end

data/spec/support/test_input.rb

@@ -0,0 +1,36 @@
+class TestInput
+
+  AskAnswer = Struct.new(:question, :answer)
+
+  attr_reader :answers
+  attr_reader :history
+
+  def initialize(answers = {})
+    @answers = answers
+    @history = []
+  end
+
+  def add_answer(msg_match, answer)
+    answers[msg_match] = answer
+  end
+
+  def asked?(msg)
+    history.any? { |h| msg === h.question }
+  end
+
+  def call(msg)
+    a = answers.detect do |k, v|
+      k === msg
+    end
+
+    value = a ? a[1] : nil
+
+    if value && value.respond_to?(:call)
+      value = value.call
+    end
+
+    history << AskAnswer.new(msg, value)
+    value
+  end
+
+end

data/spec/unit/cancel_command_spec.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::CancelCommand do
+
+  let(:options) { DoiExtractor::Options.for_command('cancel', project: 'cps', environment: 'local', api_password: 'abc123', force: true, extract_group_id: 1).tap {|o| o.validate } }
+  let(:output) { StringIO.new }
+  let(:command) { DoiExtractor::CancelCommand.new(options, output) }
+
+  let(:get_url) { 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1' }
+  let(:cancel_url) { 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1/cancel' }
+
+  context 'when successful' do
+
+    let!(:get_stub) { stub_request(:get, get_url).to_return(body: '{"id": 1, "status": "submitted"}') }
+    let!(:cancel_stub) { stub_request(:post, cancel_url).to_return(body: '{"id": 1, "status": "cancelled"}') }
+
+    it 'makes two API calls' do
+      command.execute
+      expect(get_stub).to have_been_requested.times(1)
+      expect(cancel_stub).to have_been_requested.times(1)
+    end
+
+    it 'writes to sdtout' do
+      command.execute
+      expect(output.string).to match /complete/i
+    end
+  end
+end

data/spec/unit/command_line_parser_spec.rb

@@ -0,0 +1,68 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::CommandLineParser do
+
+  shared_examples 'command_examples' do |name, extra_flags = []|
+    let(:parser) { DoiExtractor::CommandLineParser.parse([name] + %w(-p cps -e live) + extra_flags) }
+
+    it 'should be valid' do
+      expect(parser.valid?).to eq true
+    end
+
+    it "should contain options" do
+      expect(parser.options).to be_a DoiExtractor::Options
+    end
+
+    it 'should set options attributes' do
+      expect(parser.options.command).to eq name
+      expect(parser.options.project).to eq 'cps'
+      expect(parser.options.environment).to eq 'live'
+    end
+  end
+
+  context 'with a valid create command' do
+    include_examples 'command_examples', 'create', %w(-v 1.1)
+  end
+
+  context 'with a valid create command with email' do
+    include_examples 'command_examples', 'create', %w(-v 1.2 -E roland@tet.com)
+
+    it 'should set the email option' do
+      puts parser.options.inspect
+      expect(parser.options.email).to eq 'roland@tet.com'
+    end
+  end
+
+  context 'with a valid status command' do
+    include_examples 'command_examples', 'status', %w(-i 1)
+  end
+
+  context 'with a valid download command' do
+    include_examples 'command_examples', 'download', %w(-i 1)
+  end
+
+  context 'with an invalid command' do
+    let(:parser) { DoiExtractor::CommandLineParser.parse(%w(invalid -p cps)) }
+
+    it 'should not be valid' do
+      expect(parser.valid?).to eq false
+    end
+
+    it 'should have a reasonable error message' do
+      expect(parser.to_s).to match(/invalid\s+command/i)
+    end
+
+    it 'should not have any options' do
+      expect(parser.options).to be_nil
+    end
+  end
+
+  context 'with an unknown option' do
+    let(:parser) { DoiExtractor::CommandLineParser.parse(%w(status -p cps -e live -x extra))}
+
+    it 'should not be valid' do
+      expect(parser.valid?).to eq false
+    end
+  end
+
+end

data/spec/unit/create_command_spec.rb

@@ -0,0 +1,44 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::CreateCommand do
+
+  let(:options) { DoiExtractor::Options.for_command('create', project: 'cps', environment: 'local', api_password: 'abc123').tap {|o| o.validate } }
+  let(:output) { StringIO.new }
+  let(:command) { DoiExtractor::CreateCommand.new(options, output) }
+
+  let(:recent_url) { 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/recent?type=doi' }
+  let(:create_url) { 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups' }
+
+  context 'when successful' do
+
+    let!(:recent_stub) { stub_request(:get, recent_url).to_return(body: '[]') }
+    let!(:create_stub) { stub_request(:post, create_url).to_return(body: '{"id": 1, "status": "empty", "doi_version": "1.1", "year": "2017", "extract_requests": [{"id": 10}, {"id": 11}]}') }
+
+    it 'makes 2 API calls' do
+      command.execute
+      expect(recent_stub).to have_been_requested.times(1)
+      expect(create_stub).to have_been_requested.times(1)
+    end
+
+    it 'writes to sdtout' do
+      command.execute
+      expect(output.string).to match /doi extract with id \d+ submitted./im
+    end
+  end
+
+  context 'when an existing extract is processing' do
+    let!(:recent_stub) { stub_request(:get, recent_url).to_return(body: '[{"id": 1, "status": "submitted"}]') }
+    let!(:create_stub) { stub_request(:post, create_url) }
+
+    it 'makes 1 API call' do
+      command.execute rescue DoiExtractor::CommandFailError
+      expect(recent_stub).to have_been_requested.times(1)
+      expect(create_stub).not_to have_been_requested
+    end
+
+    it 'raises a CommandFailError' do
+      expect { command.execute }.to raise_error DoiExtractor::CommandFailError
+    end
+  end
+
+end

data/spec/unit/download_command_spec.rb

@@ -0,0 +1,139 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::DownloadCommand do
+
+  let(:base_path) { File.expand_path(File.dirname(__FILE__) + '/../testdata/') }
+  let(:path) { File.join(base_path, '2017_v1.1') }
+  let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'local', extract_group_id: 1, api_password: 'abc123', download_base_path: base_path, force: true).tap {|o| o.validate } }
+  let(:output) { StringIO.new }
+  let(:input) { TestInput.new }
+  let(:command) { DoiExtractor::DownloadCommand.new(options, output, input) }
+
+  let(:complete_extract_group_json) {
+    {
+        id: 1,
+        year: '2017',
+        doi_version: '1.1',
+        status: 'complete',
+        all_data_available: true,
+        citation: 'this is a citation',
+        codebook_extract: {id: 40, extract_file_name: 'cps_004.dat.gz', samples: [{id: 100, name: '1990a'}, {id: 200, name: '2000a'}, {id: 300, name: '2010a'}], files: [{filename: 'cps_004.cbk', ext: 'cbk'}, {filename: 'cps_004.xml', ext: 'xml'}]},
+        extract_requests: [
+            {id: 10, extract_file_name: 'cps_001.dat.gz', samples: [{id: 100, name: '1990a'}], files: [{filename: 'cps_001.dat.gz', ext: 'dat.gz'}, {filename: 'cps_001.cbk', ext: 'cbk'}, {filename: 'cps_001.xml', ext: 'xml'}]},
+            {id: 20, extract_file_name: 'cps_002.dat.gz', samples: [{id: 200, name: '2000a'}], files: [{filename: 'cps_002.dat.gz', ext: 'dat.gz'}, {filename: 'cps_002.cbk', ext: 'cbk'}, {filename: 'cps_002.xml', ext: 'xml'}]},
+            {id: 30, extract_file_name: 'cps_003.dat.gz', samples: [{id: 300, name: '2010a'}], files: [{filename: 'cps_003.dat.gz', ext: 'dat.gz'}, {filename: 'cps_003.cbk', ext: 'cbk'}, {filename: 'cps_003.xml', ext: 'xml'}]}
+        ]
+    }
+  }
+
+  let(:submitted_extract_group_json) {
+    {
+        id: 2,
+        year: '2017',
+        doi_version: '1.1',
+        status: 'submitted',
+        extract_requess: []
+    }
+  }
+
+  def stub_get_extract_group(id, group_json)
+    url = "http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/#{id}"
+    if group_json
+      stub_request(:get, url).to_return(body: JSON.dump(group_json))
+    else
+      stub_request(:get, url).to_return(status: 404)
+    end
+  end
+
+  before do
+    if Dir.exist?(path)
+      FileUtils.rm_rf(path)
+    end
+  end
+
+  context 'when an extract is available' do
+    let!(:get_stub) { stub_get_extract_group(1, complete_extract_group_json) }
+
+    def stub_file_download(er, f)
+      stub_request(:get, "http://localhost:3003/cps-action/api/extracts/v1/extract_requests/#{er[:id]}/file/#{f[:ext]}").to_return(body: f[:filename])
+    end
+
+    let!(:download_stubs) {
+      [].tap do |stubs|
+        complete_extract_group_json[:extract_requests].each do |er|
+          er[:files].each do |f|
+            stubs << stub_file_download(er, f)
+          end
+        end
+        stubs << stub_file_download(complete_extract_group_json[:codebook_extract], complete_extract_group_json[:codebook_extract][:files].last)
+      end
+    }
+
+    it 'executes all requests' do
+      command.execute
+      expect(get_stub).to have_been_requested.times(1)
+      download_stubs.each do |s|
+        expect(s).to have_been_requested.times(1)
+      end
+    end
+
+    it 'packages everything' do
+      command.execute
+      complete_extract_group_json[:extract_requests].each do |er|
+        s = er[:samples].first[:name]
+
+        expect(File.exist?(File.join(path, "#{s}.tar.gz"))).to be_truthy
+      end
+      expect(File.exist?(File.join(path, "summary.txt"))).to be_truthy
+    end
+
+    it 'adds the citation to the summary file' do
+      command.execute
+      txt = File.read(File.join(path, "summary.txt"))
+      expect(txt).to match /this is a citation/
+    end
+
+    it 'cleans up the tmp folder' do
+      command.execute
+      expect(File.exist?(File.join(path, 'tmp'))).to be_falsey
+    end
+
+    it 'does not prompt the user when forced it true' do
+      input.add_answer(/.*/, ->{ raise 'no asking' })
+      command.execute
+    end
+
+    context 'when force is not set' do
+      let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'local', extract_group_id: 1, api_password: 'abc123', download_base_path: base_path).tap {|o| o.validate } }
+
+      before do
+        input.add_answer(/download/, 'y')
+      end
+
+      it 'asks for user input' do
+        command.execute
+        expect(input.asked?(/download/)).to be_truthy
+      end
+    end
+  end
+
+  context 'user cancels download' do
+    let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'local', extract_group_id: 1, api_password: 'abc123', download_base_path: base_path, force: false).tap {|o| o.validate } }
+    let!(:get_stub) { stub_get_extract_group(1, complete_extract_group_json) }
+
+    before do
+      input.add_answer(/download/, 'n')
+    end
+
+    it 'asks for input' do
+      expect { command.execute }.to raise_error DoiExtractor::CommandFailError, /cancelled/
+      expect(input.asked?(/download/)).to be_truthy
+    end
+
+    it 'does not create the output path' do
+      expect { command.execute }.to raise_error DoiExtractor::CommandFailError, /cancelled/
+      expect(File.exist?(path)).to be_falsey
+    end
+  end
+
+end