doi_extractor 1.0.5

data/lib/doi_extractor/ipums_uri_builder.rb

@@ -0,0 +1,51 @@
+module DoiExtractor
+  class IpumsUriBuilder
+
+    IpumsConfig = Struct.new(:base_domain, :live_prefix, :action_name)
+
+    CONFIGS = {
+        ahtus:    IpumsConfig.new('ahtusdata.org', 'www.'),
+        atus:     IpumsConfig.new('atusdata.org', 'www.'),
+        cps:      IpumsConfig.new('cps.ipums.org'),
+        dhs:      IpumsConfig.new('idhsdata.org', 'www.', 'idhs'),
+        highered: IpumsConfig.new('highered.ipums.org'),
+        nhis:     IpumsConfig.new('nhis.ipums.org'),
+        ipumsi:   IpumsConfig.new('international.ipums.org', nil, 'international'),
+        mtus:     IpumsConfig.new('mtusdata.org', 'www.'),
+        napp:     IpumsConfig.new('nappdata.org', 'www.'),
+        usa:      IpumsConfig.new('usa.ipums.org'),
+    }
+
+    attr_reader :project, :environment, :config
+
+    def initialize(project, environment)
+      @project = project.to_sym
+      @environment = environment.to_sym
+      @config = CONFIGS[@project]
+    end
+
+    def protocol
+      environment == :local ? 'http' : 'https'
+    end
+
+    def domain
+      case environment
+        when :local
+          'localhost:3003'
+        when :live
+          config.live_prefix.to_s + config.base_domain
+        else
+          environment.to_s + '.' + config.base_domain
+      end
+    end
+
+    def path
+      (config.action_name || project).to_s + '-action'
+    end
+
+    def url
+      "#{protocol}://#{domain}/#{path}"
+    end
+
+  end
+end

data/lib/doi_extractor/old_ruby_patch.rb

@@ -0,0 +1,25 @@
+
+if RUBY_VERSION.start_with?('1.9')
+
+  # Previous to 2.0 ruby doesn't define the index methods on OpenStruct.  These are used in the IpumsClient when parsing
+  # json into OpenStruct objects.
+  class OpenStruct
+    def [](key)
+      self.send(key.to_sym)
+    end
+
+    def []=(key, value)
+      self.send("#{key}=".to_sym, value)
+    end
+  end
+
+  # 1.9 doesn't implement the 422 http code
+  module Net
+    class HTTPUnprocessableEntity < HTTPClientError
+      HAS_BODY = true
+    end
+
+    HTTPResponse::CODE_TO_OBJ['422'] = HTTPUnprocessableEntity
+  end
+
+end

data/lib/doi_extractor/options.rb

@@ -0,0 +1,132 @@
+module DoiExtractor
+  class Options
+
+    PROJECTS = %w(ahtus atus cps dhs highered nhis ipumsi mtus napp usa)
+    ENVIRONMENTS = %w(local demo internal staging live)
+    COMMANDS = %w(create download status cancel)
+    
+    OPTIONS = [
+        :command,
+        :verbose,
+        :environment,
+        :project,
+        :api_uri,
+        :api_username,
+        :api_password,
+        :download_base_path,
+        :doi_version,
+        :year,
+        :email,
+        :force,
+        :extract_group_id
+    ]
+
+    def self.for_command(cmd, attrs = nil)
+      if COMMANDS.include? cmd
+        new({command: cmd}.merge(attrs || {}))
+      else
+        raise 'Invalid Command'
+      end
+    end
+
+    attr_reader :errors
+    attr_accessor *OPTIONS
+
+    def initialize(attrs = nil)
+      assign_attrs(attrs, OPTIONS)
+    end
+
+    def validate
+      @errors = []
+
+      if environment.nil?
+        self.environment = 'live'
+      end
+
+      if !COMMANDS.include?(command)
+        errors << 'Invalid Command'
+      end
+
+      if !ENVIRONMENTS.include?(environment)
+        errors << 'Invalid Environment'
+      end
+
+      if !PROJECTS.include?(project)
+        errors << 'Invalid Project'
+      end
+
+      # if command, env or project are invalid, stop performing further validation
+      return if errors.any?
+
+      self.api_uri = calculate_api_uri if api_uri.nil?
+      errors << 'Missing API URI' if self.api_uri.nil?
+
+      # For demo, internal, and staging envs, use the basic auth creds by default
+      if %w(demo internal staging).include?(environment)
+        self.api_username ||= 'hcp'
+        self.api_password ||= '1234'
+      end
+
+      if command == 'create'
+        self.year ||= Time.now.year.to_s
+
+        if self.year !~ /^\d{4}$/
+          errors << 'Year must be a 4 digit year'
+        end
+
+        if m = /[vV]?(\d+\.\d+)/.match(self.doi_version)
+          self.doi_version = m[1]
+        else
+          errors << 'DOI version is required and must match the pattern VX.X'
+        end
+      end
+
+      if %w(download status).include?(command)
+        self.download_base_path ||= calculate_download_base_path
+
+        if self.download_base_path.nil? || self.download_base_path.strip.empty?
+          errors << 'Unable to determine download base path'
+        end
+      end
+
+      if %w(download cancel).include?(command)
+        unless self.extract_group_id
+          errors << 'Missing extract request group id'
+        end
+      end
+    end
+
+    def valid?
+      validate
+      errors.empty?
+    end
+
+    private
+
+    def calculate_api_uri
+      if project && environment
+        IpumsUriBuilder.new(project, environment).url
+      else
+        nil
+      end
+    end
+
+    def calculate_download_base_path
+      if project && environment
+        # "/pkg/ipums/programming/#{project}/#{environment}"
+        "/pkg/ipums/programming/doi_extractor/temp_archive/#{project}/#{environment}"
+      else
+        nil
+      end
+    end
+
+    def assign_attrs(attrs, known_keys)
+      if attrs
+        known_keys.each do |k|
+          self.send("#{k}=", attrs[k]) if attrs[k]
+        end
+      end
+    end
+  end
+end
+

data/lib/doi_extractor/secrets.rb

@@ -0,0 +1,18 @@
+module DoiExtractor
+  class Secrets
+    DEFAULT_API_PATH = '/pkg/mpctools/secrets/api_creds.yml'
+    DEFAULT_API_USERNAME = 'terrapop' # 'doi_extractor'
+
+    attr_reader :api_path, :api_username, :api_password
+
+    def initialize(overrides = {})
+      @api_path = overrides[:api_path] || DEFAULT_API_PATH
+      @api_username = overrides[:api_username] || DEFAULT_API_USERNAME
+      @api_password = overrides[:api_password] || read_yaml(@api_path)[@api_username]
+    end
+
+    def read_yaml(path)
+      YAML.load(ERB.new(IO.read(path)).result)
+    end
+  end
+end

data/lib/doi_extractor/status_command.rb

@@ -0,0 +1,62 @@
+module DoiExtractor
+  class StatusCommand < Command
+
+    def _execute
+      if options.extract_group_id
+        report_group(options.extract_group_id)
+      else
+        report_recent
+      end
+    end
+
+    def report_group(id)
+      doi_extract = ipums_client.get_doi_extract(id)
+
+      unless doi_extract
+        fail("no extract for #{id} found")
+      end
+
+      location = DownloadLocation.new(options.download_base_path, doi_extract)
+
+      say_doi_summary(doi_extract, location)
+      say("Extract Requests:")
+
+      fmt = '%-10s%25s%20s%12s'
+
+      say(sprintf(fmt.sub('i', 's'), 'ID', 'Status', 'Sample', 'Downloaded'))
+      doi_extract.extract_requests.each do |er|
+        say(sprintf(fmt, er.id, er.completion_status, er.samples.first.name, location.complete_extract_request?(er)))
+      end
+
+    end
+
+    def report_recent
+      recent = ipums_client.recent_doi_extracts
+
+      if recent.nil? || recent.empty?
+        say("There are no DOI extracts")
+      else
+        recent.each do |doi|
+          location = DownloadLocation.new(options.download_base_path, doi)
+          say_doi_summary(doi, location)
+          say("\tExtracts:       #{doi.extract_requests.count}")
+        end
+      end
+    end
+
+    def say_doi_summary(doi_extract, location)
+      count_hsh = Hash.new { |h, k| h[k] = 0 }
+      extract_counts = doi_extract.extract_requests.reduce(count_hsh) { |h, e| h[e.completion_status] += 1; h }
+
+      say("DOI Extract (ID: #{doi_extract.id})")
+      say("\tStatus:         #{doi_extract.status}")
+      say("\tSubmitted:      #{doi_extract.submit_date}")
+      say("\tFinished:       #{doi_extract.finish_date}")
+      say("\tData Available: #{doi_extract.all_data_available}")
+      say("\tPath:           #{location.path}")
+      say("\tExtract Counts: #{extract_counts.map { |k, v| "#{k}: #{v}" }.join(', ')}")
+      say("\tDownloaded:     #{doi_extract.extract_requests.any? && doi_extract.extract_requests.all? { |er| location.complete_extract_request?(er) }}")
+    end
+
+  end
+end

data/lib/doi_extractor/version.rb

@@ -0,0 +1,3 @@
+module DoiExtractor
+  VERSION = '1.0.5'
+end

data/spec/fixtures/api_creds.yml

@@ -0,0 +1,2 @@
+terrapop: testPassword
+doi_extractor: testPassword

data/spec/reports/SPEC-DoiExtractor-CancelCommand-when-successful.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CancelCommand when successful" tests="2" time="0.01088" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:52-05:00">
+  <testcase name="DoiExtractor::CancelCommand when successful makes two API calls" time="0.005551">
+  </testcase>
+  <testcase name="DoiExtractor::CancelCommand when successful writes to sdtout" time="0.004845">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CancelCommand.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CancelCommand" tests="0" time="0.000107" skipped="0" timestamp="2017-09-12T12:47:52-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command-with-email.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with a valid create command with email" tests="4" time="0.009902" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command with email should be valid" time="0.002419">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command with email should contain options" time="0.002116">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command with email should set options attributes" time="0.002968">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command with email should set the email option" time="0.00192">
+  </testcase>
+  <system-out>
+#&lt;DoiExtractor::Options:0x007ff44e85cdd0 @command="create", @environment="live", @project="cps", @errors=[], @api_uri="https://cps.ipums.org/cps-action", @doi_version="1.2", @year="2017", @email="roland@tet.com"&gt;
+  </system-out>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-create-command.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with a valid create command" tests="3" time="0.010797" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:52-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command should be valid" time="0.003975">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command should contain options" time="0.003727">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid create command should set options attributes" time="0.002684">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-download-command.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with a valid download command" tests="3" time="0.007531" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with a valid download command should be valid" time="0.00285">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid download command should contain options" time="0.00228">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid download command should set options attributes" time="0.001983">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-a-valid-status-command.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with a valid status command" tests="3" time="0.007673" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with a valid status command should be valid" time="0.001887">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid status command should contain options" time="0.001957">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with a valid status command should set options attributes" time="0.00351">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-invalid-command.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with an invalid command" tests="3" time="0.013999" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with an invalid command should not be valid" time="0.003411">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with an invalid command should have a reasonable error message" time="0.003179">
+  </testcase>
+  <testcase name="DoiExtractor::CommandLineParser with an invalid command should not have any options" time="0.006824">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser-with-an-unknown-option.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser with an unknown option" tests="1" time="0.005251" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CommandLineParser with an unknown option should not be valid" time="0.00477">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CommandLineParser.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CommandLineParser" tests="0" time="6.8e-05" skipped="0" timestamp="2017-09-12T12:47:52-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CreateCommand-when-an-existing-extract-is-processing.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CreateCommand when an existing extract is processing" tests="2" time="0.005917" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CreateCommand when an existing extract is processing makes 1 API call" time="0.002311">
+  </testcase>
+  <testcase name="DoiExtractor::CreateCommand when an existing extract is processing raises a CommandFailError" time="0.003323">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CreateCommand-when-successful.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CreateCommand when successful" tests="2" time="0.01331" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::CreateCommand when successful makes 2 API calls" time="0.008916">
+  </testcase>
+  <testcase name="DoiExtractor::CreateCommand when successful writes to sdtout" time="0.004134">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-CreateCommand.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::CreateCommand" tests="0" time="8.4e-05" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-DownloadCommand-user-cancels-download.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::DownloadCommand user cancels download" tests="2" time="0.008309" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::DownloadCommand user cancels download asks for input" time="0.005059">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadCommand user cancels download does not create the output path" time="0.002841">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available-when-force-is-not-set.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::DownloadCommand when an extract is available when force is not set" tests="1" time="0.060445" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available when force is not set asks for user input" time="0.060068">
+  </testcase>
+</testsuite>

data/spec/reports/SPEC-DoiExtractor-DownloadCommand-when-an-extract-is-available.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="DoiExtractor::DownloadCommand when an extract is available" tests="5" time="0.338937" failures="0" errors="0" skipped="0" timestamp="2017-09-12T12:47:53-05:00">
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available executes all requests" time="0.081153">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available packages everything" time="0.069818">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available adds the citation to the summary file" time="0.064809">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available cleans up the tmp folder" time="0.060621">
+  </testcase>
+  <testcase name="DoiExtractor::DownloadCommand when an extract is available does not prompt the user when forced it true" time="0.061706">
+  </testcase>
+</testsuite>