doi_extractor 1.0.5

data/lib/doi_extractor/command_line_parser.rb

@@ -0,0 +1,154 @@
+module DoiExtractor
+  class CommandLineParser
+
+    attr_reader :top_level_parser, :command_parsers, :command_parser, :options, :error
+
+    def initialize
+      @command_parsers = {
+          'status' => OptionParser.new do |opts|
+            opts.banner = "Usage: #{File.basename($0)} status [options]"
+            opts.version = VERSION
+            default_opts(opts)
+            api_opts(opts)
+            path_opts(opts)
+            extract_group_opts(opts)
+          end,
+
+          'create' => OptionParser.new do |opts|
+            opts.banner = "Usage: #{File.basename($0)} create [options]"
+            opts.version = VERSION
+            default_opts(opts)
+            api_opts(opts)
+
+            opts.on(:REQUIRED, '-v', '--doi-version DOI_VERSION', 'DOI version') do |v|
+              self.options.doi_version = v
+            end
+
+            opts.on('-y', '--year YEAR', 'Specify year (defaults to current year)') do |v|
+              self.options.year = v
+            end
+
+            opts.on('-E', '--email EMAIL', 'Specify email address to notify when all extracts are complete') do |v|
+              self.options.email = v
+            end
+          end,
+
+          'download' => OptionParser.new do |opts|
+            opts.banner = "Usage: #{File.basename($0)} download [options]"
+            opts.version = VERSION
+            default_opts(opts)
+            api_opts(opts)
+            path_opts(opts)
+            extract_group_opts(opts)
+            force_opts(opts, 'Downloads')
+          end,
+
+          'cancel' => OptionParser.new do |opts|
+            opts.banner = "Usage: #{File.basename($0)} cancel [options]"
+            opts.version = VERSION
+            default_opts(opts)
+            api_opts(opts)
+            extract_group_opts(opts)
+            force_opts(opts, 'Cancels')
+          end
+      }
+
+      @top_level_parser = OptionParser.new do |opts|
+        opts.version = VERSION
+        opts.banner = "Usage: #{File.basename($0)} <COMMAND> [options]\nAvailable Commands: #{@command_parsers.keys.join(', ')}\n\nSee '#{File.basename($0)} <COMMAND> --help' for more information on a specific command"
+      end
+    end
+
+    def parse(args)
+      begin
+        @top_level_parser.order!(args)
+
+        command = args.shift.to_s.downcase
+        @command_parser = @command_parsers[command]
+
+        if @command_parser
+          @options = Options.for_command(command)
+          @command_parser.order!(args)
+          if @options.valid?
+            @valid = true
+          else
+            @valid = false
+            @error = @options.errors.join("\n")
+          end
+        else
+          @valid = false
+          @error = 'Invalid Command'
+        end
+      rescue OptionParser::ParseError => err
+        @error = err
+        @valid = false
+      end
+    end
+
+    def valid?
+      @valid
+    end
+
+    def to_s
+      [error, @command_parser || @top_level_parser].compact.join("\n\n")
+    end
+
+    private
+
+    def default_opts(opts)
+      opts.on('-e', '--environment ENVIRONMENT', Options::ENVIRONMENTS, "Select environment (defaults to live; values: #{Options::ENVIRONMENTS.join(', ')})") do |v|
+        self.options.environment = v
+      end
+
+      opts.on('-p', '--project PROJECT', Options::PROJECTS, "Select project (required; values: #{Options::PROJECTS.join(', ')})") do |v|
+        self.options.project = v
+      end
+
+      opts.on('-V', '--[no-]verbose', 'Run verbosely') do |v|
+        self.options.verbose = v
+      end
+    end
+
+    def api_opts(opts)
+      opts.on('-a', '--api-path URI', URI, 'API path (calculated from env and project by default)') do |v|
+        self.options.api_uri = v
+      end
+
+      opts.on('--api-user USER', 'API username (Reads config from /pkg by default)') do |v|
+        self.options.api_username = v
+      end
+
+      opts.on('--api-password PASSWORD', 'API password (Reads config from /pkg by default)') do |v|
+        self.options.api_password = v
+      end
+    end
+
+    def path_opts(opts)
+      opts.on('-b', '--download-base-path PATH', 'Override download path') do |v|
+        self.options.download_base_path = v
+      end
+    end
+
+    def extract_group_opts(opts)
+      opts.on('-i', '--id ID', Integer, 'DOI Extract ID') do |v|
+        self.options.extract_group_id = v
+      end
+    end
+
+    def force_opts(opts, verb)
+      opts.on('-f', '--force', "#{verb} without prompting") do |v|
+        self.options.force = v
+      end
+    end
+
+    class << self
+      private :new
+
+      def parse(args)
+        opts = new
+        opts.parse(args.dup)
+        opts
+      end
+    end
+  end
+end

data/lib/doi_extractor/create_command.rb

@@ -0,0 +1,32 @@
+module DoiExtractor
+  class CreateCommand < Command
+
+    def _execute
+
+      say('checking existing DOI extracts', true)
+      if any_current_doi_extracts?
+        fail('A DOI Extract is already in progress.  Nothing created.')
+      end
+
+      say('Creating DOI extract group...')
+      ex = ipums_client.create_doi_extract(options.doi_version, options.year, options.email)
+
+      fail('Server did not return any data; extract status is unknown') unless ex
+      fail("Server created DOI extract with ID #{ex.id} and invalid status #{ex.status}") unless ex.status == 'empty'
+
+      say("DOI Extract with ID #{ex.id} submitted.  To check status run the following command:\n    doi_extractor status -p #{options.project} -e #{options.environment} -i #{ex.id}\n")
+
+      if options.email
+        say("An email will be sent to #{options.email} after the last extract has finished.  At that time you will be able to download and archive the extracts.")
+      end
+
+    end
+
+    def any_current_doi_extracts?
+      in_progress_status = ['empty', 'waiting_to_populate', 'populating', 'unsubmitted', 'waiting_to_submit', 'submitting', 'submitted']
+      recent = ipums_client.recent_doi_extracts
+      recent.any? { |e| in_progress_status.include?(e.status) }
+    end
+
+  end
+end

data/lib/doi_extractor/download_command.rb

@@ -0,0 +1,109 @@
+module DoiExtractor
+  class DownloadCommand < Command
+
+    def _execute
+      say('retrieving DOI extract', true)
+      doi_extract = ipums_client.get_doi_extract(options.extract_group_id)
+
+      unless doi_extract
+        fail("no DOI extract with ID #{id} found")
+      end
+
+      unless doi_extract.all_data_available
+        fail('The specified DOI extract group is not ready to download')
+      end
+
+      location = DownloadLocation.new(options.download_base_path, doi_extract)
+
+      if options.force
+        say('skipping user confirmation', true)
+      else
+        i = ask("download DOI extract files to #{location.path}?")
+        if i.to_s.downcase[0...1] != 'y'
+          fail('download cancelled')
+        end
+      end
+
+      location.ensure!
+
+      doi_extract.extract_requests.each_with_index do |er, idx|
+
+        if location.complete_extract_request?(er)
+          say("ER #{er.id} already downloaded")
+        else
+          say_nb("ER #{er.id} downloading (#{idx + 1} of #{doi_extract.extract_requests.count})...")
+          location.clean_tmp!(er)
+          er.files.each do |f|
+            save_path = File.join(location.extract_request_tmp_path(er), f.filename)
+            ipums_client.download_extract_file(save_path, er.id, f.ext)
+
+            # Special handling of ddi files
+            if f.ext.downcase == 'xml'
+              FileUtils.cp(save_path, location.ddi_path)
+            end
+
+          end
+          say(location.package_tmp_to_extract(er), true)
+          say("Complete")
+        end
+      end
+
+      location.clean_tmp!
+
+      codebook_er = doi_extract.codebook_extract
+      full_ddi = codebook_er ? codebook_er.files.detect { |f| f.ext == 'xml' } : nil
+
+      if full_ddi.nil?
+        fail('Missing full DDI codebook')
+      end
+
+      ipums_client.download_extract_file(location.complete_ddi_path, codebook_er.id, 'xml')
+
+
+      # Write summary file to output location
+      File.open(location.summary_file_path, 'w') do |f|
+        f.puts "File generated on #{Time.now.strftime('%c')}"
+        f.puts
+        f.puts "Extract Request Group ID #{doi_extract.id}"
+        f.puts "DOI Version v#{doi_extract.doi_version}"
+        f.puts
+        f.puts "Citation:"
+        f.puts doi_extract.citation
+        f.puts
+        f.puts 'Included Extracts:'
+
+        write_extract_summary(f, location, doi_extract)
+      end
+
+      location.remove_tmp
+
+    end
+
+    def write_extract_summary(file, location, doi_extract)
+
+      headers = ['ID', 'Sample', 'Archive Filename', 'Data Filename']
+      spacer = 4
+
+      extract_data = doi_extract.extract_requests.map do |er|
+        [er.id.to_s, location.extract_request_name(er), location.extract_request_filename(er), er.extract_file_name]
+      end
+
+      max_lengths = (extract_data + [headers]).reduce([0, 0, 0, 0]) do |lengths, data|
+        lengths.zip(data).map { |l, d| [l, d.length].max }
+      end
+
+      fmt = "%-#{max_lengths.first + spacer}s"
+      max_lengths[1..-1].each do |l|
+        fmt << "%#{l + spacer}s"
+      end
+
+      file.puts sprintf(fmt, *headers)
+
+      extract_data.each do |d|
+        file.puts sprintf(fmt, *d)
+      end
+
+    end
+
+  end
+end

data/lib/doi_extractor/download_location.rb

@@ -0,0 +1,86 @@
+module DoiExtractor
+  class DownloadLocation
+
+    attr_reader :path
+
+    def initialize(base_path, extract_group)
+      @extract_group = extract_group
+      @base_path = base_path
+      if extract_group.nil? || extract_group.year.nil? || extract_group.doi_version.nil?
+        raise 'Cannot calculate path: missing doi extract data'
+      end
+      @path = File.join(base_path, "#{extract_group.year}_v#{extract_group.doi_version}")
+    end
+
+    def exist?
+      Dir.exist? path
+    end
+
+    def ensure!
+      FileUtils.mkdir_p path
+      FileUtils.mkdir_p ddi_path
+    end
+
+    def tmp_path
+      File.join(path, 'tmp')
+    end
+
+    def remove_tmp
+      if Dir.exist?(tmp_path)
+        FileUtils.rm_rf tmp_path
+      end
+    end
+
+    def clean_tmp!(er = nil)
+      remove_tmp
+
+      FileUtils.mkdir_p tmp_path
+      if er
+        FileUtils.mkdir_p extract_request_tmp_path(er)
+      end
+    end
+
+    def summary_file_path
+      File.join(path, 'summary.txt')
+    end
+
+    def complete_ddi_path
+      File.join(ddi_path, 'complete.xml')
+    end
+
+    def ddi_path
+      File.join(path, 'ddi')
+    end
+
+    def package_tmp_to_extract(er)
+      cmd = %[tar -czvf "#{extract_request_path(er)}" -C "#{tmp_path}" "#{extract_request_name(er)}"]
+      output = `#{cmd} 2>&1`
+      raise "error running command: #{cmd}\n#{output}" unless $?.success?
+      output
+    end
+
+    def extract_request_tmp_path(er)
+      File.join(tmp_path, extract_request_name(er))
+    end
+
+
+    def complete_extract_request?(er)
+      file = extract_request_path(er)
+      File.exist?(file)
+    end
+
+    def extract_request_path(er)
+      File.join(path, extract_request_filename(er))
+    end
+
+    def extract_request_filename(er)
+      "#{extract_request_name(er)}.tar.gz"
+    end
+
+    def extract_request_name(er)
+      s = er.samples.first
+      [s.name, s.long_description].compact.join(' - ')
+    end
+
+  end
+end

data/lib/doi_extractor/errors.rb

@@ -0,0 +1,34 @@
+module DoiExtractor
+
+  class CommandFailError < StandardError
+
+    attr_reader :inner_exception, :exit_code
+
+    def initialize(message, inner_exception = nil, exit_code = 1)
+      super(message)
+      @inner_exception = inner_exception
+      @exit_code = exit_code
+    end
+
+    def error_report
+      report = []
+
+      report << "Failed: #{self.message}"
+
+      if inner_exception
+        report << "caused by: #{inner_exception.class}: #{inner_exception.message}"
+        inner_exception.backtrace.each do |l|
+          report << "    #{l}"
+        end
+      end
+
+      report << ' '
+
+      report.join("\n")
+    end
+  end
+
+  class ApiError < StandardError
+  end
+
+end

data/lib/doi_extractor/ipums_client.rb

@@ -0,0 +1,159 @@
+module DoiExtractor
+  class IpumsClient
+
+    DEFAULT_READ_TIMEOUT = 120
+
+    attr_reader :uri, :username, :password
+
+    def initialize(uri, username, password)
+      case uri
+        when String
+          @uri = URI(uri)
+        when URI
+          @uri = uri
+        else
+          raise 'Invalid uri'
+      end
+
+      @username = username
+      @password = password
+    end
+
+    def create_doi_extract(doi_version, year, email)
+      res = post('api/extracts/v1/extract_request_groups', {type: 'doi', year: year, doi_version: doi_version, email: email}, 600)
+      parse_body(res)
+    end
+
+    def submit_doi_extract(id)
+      res = post("api/extracts/v1/extract_request_groups/#{id}/submit")
+      parse_body(res)
+    end
+
+    def recent_doi_extracts
+      res = get('api/extracts/v1/extract_request_groups/recent', {type: 'doi'})
+      parse_body(res)
+    end
+
+    def get_doi_extract(id)
+      res = get("api/extracts/v1/extract_request_groups/#{id}")
+      parse_body(res)
+    end
+
+    def cancel_doi_extract(id)
+      res = post("api/extracts/v1/extract_request_groups/#{id}/cancel")
+      parse_body(res)
+    end
+
+    def submit_extract(id)
+      res = post("api/extracts/v1/extract_requests/#{id}/submit")
+      parse_body(res)
+    end
+
+    def download_extract_file(file_path, extract_request_id, extension)
+      path = "api/extracts/v1/extract_requests/#{extract_request_id}/file/#{extension}"
+      File.open(file_path, 'w') do |f|
+        perform_request('get', path, nil, f, 600)
+      end
+    end
+
+    def get(path, params = nil)
+      io = StringIO.new
+      perform_request('get', path, params, io)
+      io.string
+    end
+
+    def post(path, params = nil, read_timeout = DEFAULT_READ_TIMEOUT)
+      io = StringIO.new
+      perform_request('post', path, params, io, read_timeout)
+      io.string
+    end
+
+    def perform_request(method, path, params, response_io, read_timeout = DEFAULT_READ_TIMEOUT, redirect_limit = 5)
+      raise ApiError, 'too many HTTP redirects' if redirect_limit == 0
+
+      request_uri = self.uri.dup
+      request_uri.path = '/' + combine_uri_paths(request_uri.path, path)
+
+      Net::HTTP.start(uri.host, uri.port, {use_ssl: uri.scheme == 'https'}) do |http|
+        # in seconds
+        http.read_timeout = read_timeout
+        request = build_request(method, request_uri, params)
+
+        request.basic_auth(username, password)
+        request.add_field('Accept', 'application/json')
+
+        redirect_location = nil
+
+        http.request(request) do |response|
+          case response
+            when Net::HTTPSuccess
+              response.read_body do |chunk|
+                response_io.write chunk
+              end
+            when Net::HTTPRedirection
+              redirect_location = response['location']
+            when Net::HTTPUnprocessableEntity
+              body = response.body
+              body = body && body != '' ? JSON.parse(body) : nil
+              msg = body.map { |k, v| "#{k}: #{v}" }.join(', ')
+              raise ApiError, "API Server returned an error:  #{msg}"
+            else
+              raise ApiError, "Invalid API Server Response from [#{method.to_s.upcase} #{path}]: #{response.code} - #{response.message}"
+          end
+        end
+
+        if redirect_location
+          perform_request('get', redirect_location, nil, response_io, read_timeout, redirect_limit - 1)
+        end
+
+      end
+    end
+
+    def combine_uri_paths(*parts)
+      parts.compact.map(&:to_s).map(&:strip).map { |p| p.sub(/^\/+/, '').sub(/\/+$/, '') }.join('/')
+    end
+
+    def build_request(method, request_uri, params)
+      case method.to_s.downcase
+        when 'get'
+          case params
+            when Hash
+              request_uri.query = URI.encode_www_form(params)
+            when String
+              request_uri.query = params
+            when nil
+              # Do nothing
+            else
+              raise ApiError, "Invalid GET params: #{params}"
+          end
+          Net::HTTP::Get.new(request_uri.request_uri)
+        when 'post'
+          r = Net::HTTP::Post.new(request_uri.request_uri)
+          case params
+            when Hash
+              r.set_content_type('application/json')
+              r.body = JSON.generate(params)
+            when String
+              r.set_content_type('application/json')
+              r.body = params
+            when nil
+              # Do nothing
+            else
+              raise ApiError, "Invalid POST params: #{params}"
+          end
+          r
+        else
+          raise ApiError, "HTTP method #{method} not implemented"
+      end
+    end
+
+    def parse_body(body)
+      if body && body != ''
+        JSON.parse(body, object_class: OpenStruct)
+      else
+        nil
+      end
+    end
+
+  end
+end