nightcrawler_swift 0.4.0 → 0.5.0

data/lib/nightcrawler_swift/cli.rb

@@ -1,10 +1,16 @@
 require "optparse"
 require "json"
 
+require "nightcrawler_swift/cli/opt_parser"
+require "nightcrawler_swift/cli/runner"
+require "nightcrawler_swift/cli/formatters/basic"
+require "nightcrawler_swift/cli/commands/url_for"
+
 module NightcrawlerSwift
-  class CLI
+  module CLI
     CONFIG_FILE = ".nswiftrc"
     CACHE_FILE = ".nswift_cache"
+
     COMMANDS = {
       "list" => {
         description: "Lists all files of the bucket/container. Ex: nswift list",
@@ -12,189 +18,27 @@ module NightcrawlerSwift
       },
 
       "download" => {
-        description: "Downloads a file by path. Format: nswift download <swift path> Ex: nswift download assets/robots.txt > my-robots.txt",
+        description: "Downloads a file by path. Format: nswift download <swift_path> Ex: nswift download assets/robots.txt > my-robots.txt",
         command: NightcrawlerSwift::Download
       },
 
       "upload" => {
-        description: "Uploads some file. Format: nswift upload <real path> <swift path> Ex: nswift upload assets/robots.txt robots.txt",
+        description: "Uploads some file. Format: nswift upload <real_path> <swift_path> Ex: nswift upload assets/robots.txt robots.txt",
         command: NightcrawlerSwift::Upload
       },
 
       "delete" => {
-        description: "Deletes a file by path. Format: nswift delete <swift path> Ex: nswift delete robots.txt",
+        description: "Deletes a file by path. Format: nswift delete <swift_path> Ex: nswift delete robots.txt",
         command: NightcrawlerSwift::Delete
+      },
+
+      "url-for" => {
+        description: "Returns the public url of an object. Format: nswift url-for <swift_path> Ex: nswift url-for robots.txt",
+        command: NightcrawlerSwift::CLI::UrlFor
       }
     }
 
-    attr_reader :opt_parser, :options
-    attr_accessor :argv
-
-    def initialize argv
-      @argv = argv
-      STDOUT.sync = true
-      NightcrawlerSwift.logger.formatter = lambda {|severity, datetime, progname, msg| "#{msg}\n"}
-    end
-
-    def run
-      configure_default_options
-      parse_parameters
-      @command_name = argv.shift
-      validate_command_and_options
-      execute_command if @command_name
-    end
-
-    protected
-    def command_list command
-      array = command.new.execute
-      array.each {|hash| log hash["name"]}
-    end
-
-    def command_download command
-      filepath = argv.first
-      log command.new.execute(filepath)
-    end
-
-    def command_upload command
-      realpath = argv.shift
-      swiftpath = argv.shift
-
-      uploaded = command.new.execute swiftpath, File.open(File.expand_path(realpath), "r")
-      log(uploaded ? "success" : "failure")
-    end
-
-    def command_delete command
-      filepath = argv.first
-      deleted = command.new.execute(filepath).to_json
-      log(deleted ? "success" : "failure")
-    end
-
-    def user_home_dir
-      Dir.home
-    end
-
-    def log string
-      NightcrawlerSwift.logger.info string
-    end
-
-    private
-    def configure_default_options
-      @options = OpenStruct.new
-      @options.configured = true
-      @options.default_config_file = true
-      @options.config_file = File.expand_path(File.join(user_home_dir, CONFIG_FILE))
-      @options.cache_file = File.expand_path(File.join(user_home_dir, CACHE_FILE))
-      @options.command = nil
-    end
-
-    def validate_command_and_options
-      if @command_name.nil? or argv.nil?
-        log opt_parser.help
-        exit
-      end
-
-      unless options.configured
-        log "You must configure your swift credentials, take a look at:\n   #{options.config_file}"
-        exit
-      end
-    end
-
-    def config_hash
-      @config_hash ||= JSON.parse(File.read(options.config_file))
-
-    rescue Errno::ENOENT => e
-      log "No such file or directory - #{options.config_file}"
-      exit 1
-    end
-
-    def execute_command
-      NightcrawlerSwift.configure config_hash
-      connect_and_execute do
-        if command = COMMANDS[@command_name]
-          command_module = command[:command]
-          command_method = "command_#{@command_name}"
-          self.send(command_method, command_module)
-        end
-      end
-
-    rescue Exceptions::BaseError => e
-      log e.message
-      exit 1
-    end
-
-    def connect_and_execute &block
-      path = options.cache_file
-      if File.exist?(path)
-        hash = JSON.parse File.read(path)
-        NightcrawlerSwift.connection.auth_response = OpenStruct.new(hash)
-        NightcrawlerSwift.connection.configure
-      end
-
-      begin
-        block.call
-      ensure
-        File.open(path, "w") do |f|
-          f.write(NightcrawlerSwift.connection.auth_response.to_h.to_json)
-        end
-      end
-    end
-
-    def parse_parameters
-      configure_opt_parser
-      opt_parser.parse!(argv)
-      check_rcfile if options.default_config_file
-
-    rescue OptionParser::InvalidOption => e
-      log e.message
-      exit 1
-    end
-
-    def configure_opt_parser
-      @opt_parser = OptionParser.new do |opts|
-        opts.banner = "nswift #{NightcrawlerSwift::VERSION}"
-        opts.separator "Usage: nswift command [options]"
-
-        opts.separator ""
-        opts.separator "commands:"
-        COMMANDS.keys.each do |key|
-          opts.separator "    #{key}\t\t\t     #{COMMANDS[key][:description]}"
-        end
-
-        opts.separator ""
-        opts.separator "options:"
-
-        opts.on("-c", "--config=PATH", String, "Alternative '#{CONFIG_FILE}' file") do |path|
-          path = File.expand_path(path.strip)
-          log "Using custom config file at: #{path}"
-          options.config_file = path
-          options.default_config_file = false
-        end
-
-        opts.on_tail("-h", "--help", "Show this message") do
-          log opts.help
-          exit
-        end
-
-        opts.on_tail("-v", "--version", "Show version") do
-          log NightcrawlerSwift::VERSION
-          exit
-        end
-      end
-    end
-
-    def check_rcfile
-      unless File.exist?(options.config_file)
-        File.open(options.config_file, "w") { |f|
-          f.write(sample_rcfile)
-        }
-      end
-
-      if sample_rcfile == File.read(options.config_file)
-        options.configured = false
-      end
-    end
-
-    def sample_rcfile
+    def self.sample_rcfile
       JSON.pretty_generate({
         bucket: "<bucket/container name>",
         tenant_name: "<tenant name>",
@@ -203,6 +47,5 @@ module NightcrawlerSwift
         auth_url: "<auth url, ex: https://auth.url.com:123/v2.0/tokens>"
       })
     end
-
   end
 end

data/lib/nightcrawler_swift/command.rb

@@ -11,40 +11,31 @@ module NightcrawlerSwift
       NightcrawlerSwift.options
     end
 
+    # :nocov:
     def execute
       raise NotImplemented.new
     end
+    # :nocov:
 
     protected
 
     def get url, params = {}
       prepare_params params
-      resource = resource_for url
-      resource.get(params[:headers])
+      Gateway.new(url).request {|r| r.get params[:headers]}
     end
 
     def put url, params = {}
       prepare_params params
-      resource = resource_for url
-      resource.put(params[:body], params[:headers])
+      Gateway.new(url).request {|r| r.put params[:body], params[:headers]}
     end
 
     def delete url, params
       prepare_params params
-      resource = resource_for url
-      resource.delete(params[:headers])
+      Gateway.new(url).request {|r| r.delete params[:headers]}
     end
 
     private
 
-    def resource_for url
-      RestClient::Resource.new(
-        url,
-        verify_ssl: options.verify_ssl,
-        timeout: options.timeout
-      )
-    end
-
     def prepare_params params
       params[:headers] ||= {}
       params[:headers]["X-Storage-Token"] = connection.token_id

data/lib/nightcrawler_swift/commands/delete.rb

@@ -2,14 +2,12 @@ module NightcrawlerSwift
   class Delete < Command
 
     def execute path
+      if path.nil? or path.empty?
+        raise Exceptions::ValidationError.new "Delete command requires a path parameter"
+      end
+
       response = delete "#{connection.upload_url}/#{path}", headers: {accept: :json }
       [200, 201].include?(response.code)
-
-    rescue RestClient::ResourceNotFound => e
-      raise Exceptions::NotFoundError.new(e)
-
-    rescue StandardError => e
-      raise Exceptions::ConnectionError.new(e)
     end
 
   end

data/lib/nightcrawler_swift/commands/download.rb

@@ -2,14 +2,12 @@ module NightcrawlerSwift
   class Download < Command
 
     def execute path
+      if path.nil? or path.empty?
+        raise Exceptions::ValidationError.new "Download command requires a path parameter"
+      end
+
       response = get "#{connection.public_url}/#{options.bucket}/#{path}"
       response.body
-
-    rescue RestClient::ResourceNotFound => e
-      raise Exceptions::NotFoundError.new(e)
-
-    rescue StandardError => e
-      raise Exceptions::ConnectionError.new(e)
     end
 
   end

data/lib/nightcrawler_swift/commands/list.rb

@@ -4,12 +4,6 @@ module NightcrawlerSwift
     def execute
       response = get connection.upload_url, headers: {accept: :json}
       JSON.parse(response.body)
-
-    rescue RestClient::ResourceNotFound => e
-      raise Exceptions::NotFoundError.new(e)
-
-    rescue StandardError => e
-      raise Exceptions::ConnectionError.new(e)
     end
 
   end

data/lib/nightcrawler_swift/commands/upload.rb

@@ -4,12 +4,9 @@ module NightcrawlerSwift
     def execute path, file
       content = file.read
       headers = {etag: etag(content), content_type: content_type(file)}
-      headers.merge!(cache_control: "max-age=#{options.max_age}") if options.max_age
+      headers.merge!(cache_control: "public, max-age=#{options.max_age}") if options.max_age
       response = put "#{connection.upload_url}/#{path}", body: content, headers: headers
       [200, 201].include?(response.code)
-
-    rescue RestClient::UnprocessableEntity => e
-      raise Exceptions::ValidationError.new(e)
     end
 
     private

data/lib/nightcrawler_swift/connection.rb

@@ -7,7 +7,7 @@ module NightcrawlerSwift
       authenticate!
       configure
 
-      NightcrawlerSwift.logger.debug  "[NightcrawlerSwift] Connected, token_id: #{token_id}"
+      NightcrawlerSwift.logger.debug "[NightcrawlerSwift] Connected, token_id: #{token_id}"
       self
     end
 
@@ -28,24 +28,24 @@ module NightcrawlerSwift
     end
 
     def authenticate!
-      auth_options = {
-        tenantName: opts.tenant_name,
-        passwordCredentials: {username: opts.username, password: opts.password}
-      }
-
-      resource = RestClient::Resource.new(
-        opts.auth_url,
-        verify_ssl: NightcrawlerSwift.options.verify_ssl,
-        timeout: NightcrawlerSwift.options.timeout
-      )
-
-      response = resource.post({ auth: auth_options }.to_json, content_type: :json, accept: :json)
+      url = opts.auth_url
+      headers = {content_type: :json, accept: :json}
+      response = Gateway.new(url).request {|r| r.post(auth_options.to_json, headers)}
 
       @auth_response = OpenStruct.new(JSON.parse(response.body))
     rescue StandardError => e
       raise Exceptions::ConnectionError.new(e)
     end
 
+    def auth_options
+      {
+        auth: {
+          tenantName: opts.tenant_name,
+          passwordCredentials: {username: opts.username, password: opts.password}
+        }
+      }
+    end
+
     def select_token
       @token_id = auth_response.access["token"]["id"]
       @expires_at = auth_response.access["token"]["expires"]
@@ -63,9 +63,9 @@ module NightcrawlerSwift
     end
 
     def configure_urls
-      @admin_url = @endpoints["adminURL"]
+      @admin_url = opts.admin_url || @endpoints["adminURL"]
+      @public_url = opts.public_url || @endpoints["publicURL"]
       @upload_url = "#{@admin_url}/#{opts.bucket}"
-      @public_url = @endpoints["publicURL"]
     end
   end
 end

data/lib/nightcrawler_swift/exceptions.rb

@@ -5,12 +5,13 @@ module NightcrawlerSwift
       attr_accessor :original_exception
 
       def initialize exception
-        super(exception.message)
+        super(exception.is_a?(String) ? exception : exception.message)
         @original_exception = exception
       end
     end
 
     class ConnectionError < BaseError; end
+    class UnauthorizedError < ConnectionError; end
     class ValidationError < ConnectionError; end
     class NotFoundError < BaseError; end
     class ConfigurationError < StandardError; end

data/lib/nightcrawler_swift/gateway.rb

@@ -0,0 +1,68 @@
+module NightcrawlerSwift
+  class Gateway
+
+    attr_reader :resource, :attempts, :current_retry_time
+
+    RETRY_BLACKLIST = [
+      RestClient::Unauthorized,
+      RestClient::ResourceNotFound,
+      RestClient::UnprocessableEntity
+    ]
+
+    def initialize url
+      @url = url
+      @attempts = 0
+      @current_retry_time = 1
+      @retries = NightcrawlerSwift.options.retries
+      @max_retry_time = NightcrawlerSwift.options.max_retry_time
+
+      @resource = RestClient::Resource.new(
+        @url,
+        verify_ssl: NightcrawlerSwift.options.verify_ssl,
+        timeout: NightcrawlerSwift.options.timeout
+      )
+    end
+
+    def request &block
+      begin
+        @attempts += 1
+        block.call(resource)
+
+      rescue => e
+        raise e unless recoverable?(e)
+        wait(e) and retry
+      end
+
+    rescue RestClient::Unauthorized => e
+      raise Exceptions::UnauthorizedError.new(e)
+
+    rescue RestClient::ResourceNotFound => e
+      raise Exceptions::NotFoundError.new(e)
+
+    rescue RestClient::UnprocessableEntity => e
+      raise Exceptions::ValidationError.new(e)
+
+    rescue => e
+      raise Exceptions::ConnectionError.new(e)
+    end
+
+    private
+    def log message
+      NightcrawlerSwift.logger.debug message
+    end
+
+    def recoverable? e
+      @retries and
+      !RETRY_BLACKLIST.include?(e.class) and
+      @attempts <= @retries
+    end
+
+    def wait e
+      number = "#{@attempts}/#{@retries}"
+      log "Attempt #{number} to call '#{@url}', waiting #{@current_retry_time}s and retrying. Error: #{e.message}"
+      sleep @current_retry_time
+      @current_retry_time = [@current_retry_time * 2, @max_retry_time].min
+    end
+
+  end
+end

data/lib/nightcrawler_swift/version.rb

@@ -1,3 +1,3 @@
 module NightcrawlerSwift
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/nightcrawler_swift.rb

@@ -6,8 +6,9 @@ require "multi_mime"
 require "rest_client"
 require "nightcrawler_swift/version"
 require "nightcrawler_swift/exceptions"
-require "nightcrawler_swift/command"
+require "nightcrawler_swift/gateway"
 require "nightcrawler_swift/connection"
+require "nightcrawler_swift/command"
 require "nightcrawler_swift/commands/upload"
 require "nightcrawler_swift/commands/download"
 require "nightcrawler_swift/commands/list"
@@ -36,8 +37,12 @@ module NightcrawlerSwift
     # - verify_ssl (optional, default: false)
     # - timeout (in seconds. Optional, default: nil)
     #
+    # - retries (default: 3)
+    # - max_retry_time (in seconds, default: 30)
+    #
     def configure opts = {}
-      @options = OpenStruct.new({verify_ssl: false}.merge(opts))
+      defaults = {verify_ssl: false, retries: 5, max_retry_time: 30}
+      @options = OpenStruct.new(defaults.merge(opts))
 
       if @options.max_age and not @options.max_age.is_a?(Numeric)
         raise Exceptions::ConfigurationError.new "max_age should be an Integer"

data/nightcrawler_swift.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "codeclimate-test-reporter"
   spec.add_development_dependency "byebug"
 end

data/spec/lib/nightcrawler_swift/cli/commands/url_for_spec.rb

@@ -0,0 +1,34 @@
+require "spec_helper"
+require "nightcrawler_swift/cli"
+
+describe NightcrawlerSwift::CLI::UrlFor do
+
+  let(:connection) { NightcrawlerSwift::Connection.new }
+  let(:token) { "token" }
+  let(:expires_at) { (DateTime.now + 60).to_time }
+  let(:public_url) { "server-url" }
+  let(:bucket) { "rogue" }
+
+  subject do
+    NightcrawlerSwift::CLI::UrlFor.new
+  end
+
+  before do
+    allow(NightcrawlerSwift).to receive(:connection).and_return(connection)
+    allow(connection).to receive(:token_id).and_return(token)
+    allow(connection).to receive(:expires_at).and_return(expires_at)
+    allow(connection).to receive(:public_url).and_return(public_url)
+    NightcrawlerSwift.configure bucket: bucket
+  end
+
+  describe "#execute" do
+    let :execute do
+      subject.execute "file_path"
+    end
+
+    it "returns the public url of the given path" do
+      expect(execute).to eql("server-url/#{bucket}/file_path")
+    end
+  end
+
+end

data/spec/lib/nightcrawler_swift/cli/formatters/basic_spec.rb

@@ -0,0 +1,117 @@
+require "spec_helper"
+require "nightcrawler_swift/cli"
+
+describe NightcrawlerSwift::CLI::Formatters::Basic do
+
+  let :config_dir do
+    File.expand_path(File.join(File.dirname(__FILE__), "../../../../fixtures"))
+  end
+
+  let :filepath do
+    "testfile.txt"
+  end
+
+  let :runner do
+    instance_double "Runner"
+  end
+
+  let :command do
+    command_class.new
+  end
+
+  subject do
+    NightcrawlerSwift::CLI::Formatters::Basic.new runner
+  end
+
+  describe "#command_list" do
+    let :result do
+      [{
+        "hash"=>"c9df50d4a29542f8b6d426a50c72b3de",
+        "last_modified"=>"2014-08-27T19:35:46.053560",
+        "bytes"=>4994,
+        "name"=>"assets/file.png",
+        "content_type"=>"image/png"
+      }]
+    end
+
+    let :command_class do
+      NightcrawlerSwift::List
+    end
+
+    it "lists all files in the bucket/container configured" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).and_return(result)
+      expect(runner).to receive(:log).with(result.first["name"])
+      subject.command_list command_class
+    end
+  end
+
+  describe "#command_download" do
+    let :command_class do
+      NightcrawlerSwift::Download
+    end
+
+    it "downloads the file" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).with(filepath).and_return("test-content")
+      expect(runner).to receive(:log).with("test-content")
+      expect(runner).to receive(:argv).and_return([filepath])
+      subject.command_download command_class
+    end
+  end
+
+  describe "#command_upload" do
+    let :realpath do
+      File.join config_dir, filepath
+    end
+
+    let :command_class do
+      NightcrawlerSwift::Upload
+    end
+
+    before do
+      File.open(realpath, "w") {|f| f.write("test") }
+    end
+
+    after do
+      File.delete(realpath) if File.exist?(realpath)
+    end
+
+    it "uploads the file" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).with(filepath, instance_of(File)).and_return(true)
+      expect(runner).to receive(:log).with("success")
+      expect(runner).to receive(:argv).twice.and_return([realpath, filepath])
+      subject.command_upload command_class
+    end
+  end
+
+  describe "#command_delete" do
+    let :command_class do
+      NightcrawlerSwift::Delete
+    end
+
+    it "deletes the file" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).with(filepath).and_return(true)
+      expect(runner).to receive(:log).with("success")
+      expect(runner).to receive(:argv).and_return([filepath])
+      subject.command_delete command_class
+    end
+  end
+
+  describe "#command_url_for" do
+    let :command_class do
+      NightcrawlerSwift::CLI::UrlFor
+    end
+
+    it "returns the public url of the given path" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).with(filepath).and_return("public-url")
+      expect(runner).to receive(:log).with("public-url")
+      expect(runner).to receive(:argv).and_return([filepath])
+      subject.command_url_for command_class
+    end
+  end
+
+end