doi_extractor 1.0.5

data/spec/unit/download_location_spec.rb

@@ -0,0 +1,71 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::DownloadLocation do
+
+  let(:base_path) { File.expand_path(File.dirname(__FILE__) + '/../testdata/') }
+  let(:path) { File.join(base_path, '2017_v1.1') }
+  let(:extract) {
+    er = OpenStruct.new
+    er.id = 5
+    er.samples = [OpenStruct.new.tap { |s| s.name = '1990a'; s.long_description = 'US 1990 5%' }]
+    er
+  }
+
+  let(:extract_group) {
+    doi = OpenStruct.new
+    doi.id = 1
+    doi.year = '2017'
+    doi.doi_version = '1.1'
+    doi.extract_requests = [extract]
+    doi
+  }
+
+  let(:location) { DoiExtractor::DownloadLocation.new(base_path, extract_group) }
+
+  after do
+    if Dir.exist?(base_path)
+      FileUtils.rm_rf(base_path)
+    end
+  end
+
+  it 'calculates extract request paths' do
+    expect(location.extract_request_path(extract)).to eq File.join(path, '1990a - US 1990 5%.tar.gz')
+  end
+
+  it 'can create itself' do
+    expect(Dir.exist?(path)).to be_falsey
+    location.ensure!
+    expect(Dir.exist?(path)).to be_truthy
+  end
+
+  it 'creates and clears the tmp path' do
+    location.ensure!
+    expect(Dir.exist?(File.join(path, 'tmp'))).to be_falsey
+    location.clean_tmp!
+    expect(Dir.exist?(File.join(path, 'tmp'))).to be_truthy
+  end
+
+  it 'packages up downloaded extract files' do
+    files = {
+        File.join(location.tmp_path, '1990a - US 1990 5%', 'cps_001.dat') => 'data file',
+        File.join(location.tmp_path, '1990a - US 1990 5%', 'cps_001.cbk') => 'basic codebook',
+        File.join(location.tmp_path, '1990a - US 1990 5%', 'cps_001.spss') => 'spss codebook contents'
+    }
+    location.ensure!
+    location.clean_tmp!(extract)
+
+    files.each do |k, v|
+      File.write(k, v)
+    end
+
+    expect(File.exist?(location.extract_request_path(extract))).to be_falsey
+
+    location.package_tmp_to_extract(extract)
+
+    expect(File.exist?(location.extract_request_path(extract))).to be_truthy
+
+    tar_files = `tar -tf "#{location.extract_request_path(extract)}"`.split("\n")
+    expect(tar_files).to contain_exactly '1990a - US 1990 5%/', *files.keys.map { |f| "1990a - US 1990 5%/#{File.basename(f)}" }
+  end
+
+end

data/spec/unit/ipums_client_spec.rb

@@ -0,0 +1,23 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::IpumsClient do
+
+  it 'handles HTTP 422' do
+
+    stub_request(:get, 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1').to_return(status: 422, body: '{"error": "bad stuff"}')
+
+    client = DoiExtractor::IpumsClient.new('http://localhost:3003/cps-action', 'user', 'pass')
+
+    expect { client.get_doi_extract(1) }.to raise_error(DoiExtractor::ApiError, /returned an error/)
+
+  end
+
+  it 'handles HTTP 500' do
+    stub_request(:get, 'http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/1').to_return(status: 500, body: '{"error": "bad stuff"}')
+
+    client = DoiExtractor::IpumsClient.new('http://localhost:3003/cps-action', 'user', 'pass')
+
+    expect { client.get_doi_extract(1) }.to raise_error(DoiExtractor::ApiError, /invalid/i)
+  end
+
+end

data/spec/unit/ipums_uri_builder_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::IpumsUriBuilder do
+
+  context 'live environment' do
+    it 'builds correct urls' do
+      expect(DoiExtractor::IpumsUriBuilder.new('cps', 'live').url).to eq 'https://cps.ipums.org/cps-action'
+      expect(DoiExtractor::IpumsUriBuilder.new('dhs', 'live').url).to eq 'https://www.idhsdata.org/idhs-action'
+    end
+  end
+
+  context 'local environment' do
+    it 'builds correct urls' do
+      expect(DoiExtractor::IpumsUriBuilder.new('cps', 'local').url).to eq 'http://localhost:3003/cps-action'
+      expect(DoiExtractor::IpumsUriBuilder.new('dhs', 'local').url).to eq 'http://localhost:3003/idhs-action'
+    end
+  end
+
+  context 'internal environment' do
+    it 'builds correct urls' do
+      expect(DoiExtractor::IpumsUriBuilder.new('cps', 'internal').url).to eq 'https://internal.cps.ipums.org/cps-action'
+      expect(DoiExtractor::IpumsUriBuilder.new('dhs', 'internal').url).to eq 'https://internal.idhsdata.org/idhs-action'
+    end
+  end
+
+end

data/spec/unit/options_spec.rb

@@ -0,0 +1,86 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::Options do
+
+  describe '.for_command' do
+
+    context 'with download command' do
+      let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'live') }
+
+      it 'returns the right class' do
+        expect(options).to be_a DoiExtractor::Options
+      end
+
+      it 'sets the attributes' do
+        expect(options.command).to eq 'download'
+        expect(options.project).to eq 'cps'
+        expect(options.environment).to eq 'live'
+      end
+    end
+
+    context 'with create command' do
+      let(:options) { DoiExtractor::Options.for_command('create', project: 'cps', environment: 'live', doi_version: '1.1') }
+
+      it 'returns the right class' do
+        expect(options).to be_a DoiExtractor::Options
+      end
+
+      it 'sets the attributes' do
+        expect(options.command).to eq 'create'
+        expect(options.project).to eq 'cps'
+        expect(options.environment).to eq 'live'
+      end
+
+      context 'with invalid doi version' do
+        let(:options) { DoiExtractor::Options.for_command('create', project: 'cps', environment: 'live', doi_version: 'asdf') }
+
+        it 'is not valid' do
+          expect(options.valid?).to be_falsey
+        end
+      end
+    end
+
+    context 'with status command' do
+      let(:options) { DoiExtractor::Options.for_command('status', project: 'cps', environment: 'live') }
+
+      it 'returns the right class' do
+        expect(options).to be_a DoiExtractor::Options
+      end
+
+      it 'sets the attributes' do
+        expect(options.command).to eq 'status'
+        expect(options.project).to eq 'cps'
+        expect(options.environment).to eq 'live'
+      end
+    end
+
+  end
+
+  context 'when setting path values' do
+    let(:options) { DoiExtractor::Options.for_command('download', project: 'cps', environment: 'live', extract_group_id: 1) }
+
+    it 'defaults the base path' do
+      expect(options.valid?).to be_truthy
+      expect(options.download_base_path).to eq "/pkg/ipums/programming/doi_extractor/temp_archive/cps/live"
+    end
+
+    it 'allows overriding the base path' do
+      options.download_base_path = '/new/place'
+      expect(options.valid?).to be_truthy
+      expect(options.download_base_path).to eq '/new/place'
+    end
+
+    it 'does not allow an empty base path' do
+      options.download_base_path = ''
+      expect(options.valid?).not_to be_truthy
+    end
+  end
+
+  it 'defaults to basic auth creds on internal env' do
+    opts = DoiExtractor::Options.for_command('download', project: 'cps', environment: 'internal', extract_group_id: 1)
+    opts.validate
+    expect(opts.api_username).to eq 'hcp'
+    expect(opts.api_password).to eq '1234'
+  end
+
+end

data/spec/unit/secrets_spec.rb

@@ -0,0 +1,14 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::Secrets do
+  it 'reads the api password' do
+    s = DoiExtractor::Secrets.new(api_path: File.expand_path(File.dirname(__FILE__) + '/../fixtures/api_creds.yml'))
+    expect(s.api_username).to eq 'terrapop'
+    expect(s.api_password).to eq 'testPassword'
+  end
+
+  it 'overrides correctly' do
+    s = DoiExtractor::Secrets.new(api_path: '/fake/not/here.yml', api_password: '123')
+    expect(s.api_password).to eq '123'
+  end
+end

data/spec/unit/status_command_spec.rb

@@ -0,0 +1,46 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DoiExtractor::StatusCommand do
+
+  let(:base_path) { File.expand_path(File.dirname(__FILE__) + '/../testdata/') }
+  let(:path) { File.join(base_path, '2017_v1.1') }
+  let(:options) { DoiExtractor::Options.for_command('status', project: 'cps', environment: 'local', extract_group_id: 1, api_password: 'abc123', download_base_path: base_path).tap {|o| o.validate } }
+  let(:output) { StringIO.new }
+  let(:input) { TestInput.new }
+  let(:command) { DoiExtractor::StatusCommand.new(options, output, input) }
+
+  let(:complete_extract_group_json) {
+    {
+        id: 1,
+        year: '2017',
+        doi_version: '1.1',
+        status: 'complete',
+        all_data_available: true,
+        citation: 'this is a citation',
+        extract_requests: [
+            {id: 10, completion_status: 'email_sent', samples: [{id: 100, name: '1990a'}], files: [{filename: 'cps_001.dat.gz', ext: 'dat.gz'}, {filename: 'cps_001.cbk', ext: 'cbk'}, {filename: 'cps_001.spss', ext: 'spss'}]},
+            {id: 20, completion_status: 'email_sent', samples: [{id: 200, name: '2000a'}], files: [{filename: 'cps_002.dat.gz', ext: 'dat.gz'}, {filename: 'cps_002.cbk', ext: 'cbk'}, {filename: 'cps_002.spss', ext: 'spss'}]}
+        ]
+    }
+  }
+
+  def stub_get_extract_group(id, group_json)
+    url = "http://localhost:3003/cps-action/api/extracts/v1/extract_request_groups/#{id}"
+    if group_json
+      stub_request(:get, url).to_return(body: JSON.dump(group_json))
+    else
+      stub_request(:get, url).to_return(status: 404)
+    end
+  end
+
+  it 'generates some output' do
+    stub = stub_get_extract_group(1, complete_extract_group_json)
+
+    command.execute
+
+    expect(output.string).to match /DOI Extract \(ID: 1/
+
+    expect(stub).to have_been_requested
+
+  end
+end

metadata

@@ -0,0 +1,282 @@
+--- !ruby/object:Gem::Specification
+name: doi_extractor
+version: !ruby/object:Gem::Version
+  version: 1.0.5
+platform: ruby
+authors:
+- Dan Elbert
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.6
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+- !ruby/object:Gem::Dependency
+  name: ci_reporter_rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+description: 
+email:
+- delbert@umn.edu
+executables:
+- doi_extractor
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Capfile
+- Gemfile
+- README.md
+- Rakefile
+- bin/doi_extractor
+- config/deploy.rb
+- config/deploy/production.rb
+- doi_extractor.gemspec
+- lib/doi_extractor.rb
+- lib/doi_extractor/cancel_command.rb
+- lib/doi_extractor/command.rb
+- lib/doi_extractor/command_line_parser.rb
+- lib/doi_extractor/create_command.rb
+- lib/doi_extractor/download_command.rb
+- lib/doi_extractor/download_location.rb
+- lib/doi_extractor/errors.rb
+- lib/doi_extractor/ipums_client.rb
+- lib/doi_extractor/ipums_uri_builder.rb
+- lib/doi_extractor/old_ruby_patch.rb
+- lib/doi_extractor/options.rb
+- lib/doi_extractor/secrets.rb
+- lib/doi_extractor/status_command.rb
+- lib/doi_extractor/version.rb
+- spec/fixtures/api_creds.yml
+- spec/reports/SPEC-DoiExtractor-CancelCommand-when-successful.xml
+- spec/reports/SPEC-DoiExtractor-CancelCommand.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command-with-email.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-download-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-status-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-invalid-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-unknown-option.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand-when-an-existing-extract-is-processing.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand-when-successful.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-user-cancels-download.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available-when-force-is-not-set.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand.xml
+- spec/reports/SPEC-DoiExtractor-DownloadLocation.xml
+- spec/reports/SPEC-DoiExtractor-IpumsClient.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-internal-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-live-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-local-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command-with-invalid-doi-version.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-download-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-status-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-when-setting-path-values.xml
+- spec/reports/SPEC-DoiExtractor-Options.xml
+- spec/reports/SPEC-DoiExtractor-Secrets.xml
+- spec/reports/SPEC-DoiExtractor-StatusCommand.xml
+- spec/spec_helper.rb
+- spec/support/test_input.rb
+- spec/unit/cancel_command_spec.rb
+- spec/unit/command_line_parser_spec.rb
+- spec/unit/create_command_spec.rb
+- spec/unit/download_command_spec.rb
+- spec/unit/download_location_spec.rb
+- spec/unit/ipums_client_spec.rb
+- spec/unit/ipums_uri_builder_spec.rb
+- spec/unit/options_spec.rb
+- spec/unit/secrets_spec.rb
+- spec/unit/status_command_spec.rb
+homepage: https://github.umn.edu/mpc/doi_extractor
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Command line tool to package IPUMS DOI extracts
+test_files:
+- spec/fixtures/api_creds.yml
+- spec/reports/SPEC-DoiExtractor-CancelCommand-when-successful.xml
+- spec/reports/SPEC-DoiExtractor-CancelCommand.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command-with-email.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-download-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-status-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-invalid-command.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-unknown-option.xml
+- spec/reports/SPEC-DoiExtractor-CommandLineParser.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand-when-an-existing-extract-is-processing.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand-when-successful.xml
+- spec/reports/SPEC-DoiExtractor-CreateCommand.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-user-cancels-download.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available-when-force-is-not-set.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available.xml
+- spec/reports/SPEC-DoiExtractor-DownloadCommand.xml
+- spec/reports/SPEC-DoiExtractor-DownloadLocation.xml
+- spec/reports/SPEC-DoiExtractor-IpumsClient.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-internal-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-live-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder-local-environment.xml
+- spec/reports/SPEC-DoiExtractor-IpumsUriBuilder.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command-with-invalid-doi-version.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-create-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-download-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command-with-status-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-for-command.xml
+- spec/reports/SPEC-DoiExtractor-Options-when-setting-path-values.xml
+- spec/reports/SPEC-DoiExtractor-Options.xml
+- spec/reports/SPEC-DoiExtractor-Secrets.xml
+- spec/reports/SPEC-DoiExtractor-StatusCommand.xml
+- spec/spec_helper.rb
+- spec/support/test_input.rb
+- spec/unit/cancel_command_spec.rb
+- spec/unit/command_line_parser_spec.rb
+- spec/unit/create_command_spec.rb
+- spec/unit/download_command_spec.rb
+- spec/unit/download_location_spec.rb
+- spec/unit/ipums_client_spec.rb
+- spec/unit/ipums_uri_builder_spec.rb
+- spec/unit/options_spec.rb
+- spec/unit/secrets_spec.rb
+- spec/unit/status_command_spec.rb