nightcrawler_swift 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 616e1624837906ed1b5381823a1de196ba8b41e7
-  data.tar.gz: 2e4d0b8eb39afb34e6cda6d5e7725c6938a3a817
+  metadata.gz: ce783e76729a865b6b3cd1fffbed84cc287234e9
+  data.tar.gz: 922f51e1130d4bb27b84bde8de018793c5b5a1c4
 SHA512:
-  metadata.gz: 193eccd937c649b468e4b24cff3728d22a6acd7de8ac824876732bff14fbb145e33dafb222ff6069e350912c849b7c55d5e75a56e25a11357e6a8e9d368ece6c
-  data.tar.gz: 466c00ebec3b67d64efca29ec820afea48ad74a0e702617dc91e570cf0c8c08bfca7a0808b2d8c8d01d3c09d398c64d7df7e7766312e1c35826fa500dd1af264
+  metadata.gz: ba0dc5e45b62d8011f321aba5c0e28487d5b496cde054a2197db3db5baeb2c9936a542ef5ef02ad9b3d324261b8677b6520140df048045ed4f579fd5bdb8db8e
+  data.tar.gz: a55710e1eddf95712cb5c3821b1853854b815ecf066ee6bf22c7b259e66f1dd7ad65f1f10405e4b8fe88b0b7c3482dbea3e05faaf9a1343fc1c7425af4152da1

data/.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
 pkg
+coverage

data/.travis.yml

@@ -1,4 +1,7 @@
 language: ruby
 rvm:
-  - 2.1.2
+- 2.1.2
 script: bundle exec rspec
+env:
+  global:
+    secure: C1/OCk7bQxlGogDpgYf0vp6e5P/+TsFvO3bnJdfUoMlO3FAZafqq+xarWNZ/caaNEkYUQ2OaQmLYKZ32yVUAPoE1mUou9vijkOIi+m/JHr1sfeJykNYZYVqi7JQ/PILw2Qs80bZ3I0AS7hWCRIADeQUPuWul06uxYEUV1SB6mQI=

data/Changelog.md

@@ -1,11 +1,21 @@
 # Changelog
 
+## 0.5.0
+
+  - CLI now supports ```-b/--bucket``` to override the default bucket/container (issue #10)
+  - CLI command for printing the public url of a given path (issue #18)
+  - CLI better treatment for errors and invalid commands
+  - Bugfix: CLI when receives a invalid command doesn't shows that it is an invalid command (issue #20)
+  - Configurable ```admin_url``` and ```public_url``` (issue #14)
+  - Included "public" directive in ```Cache-Control``` header
+  - Each request will retry X times before failing, the number of retries and the max waiting time can be configured through options
+
 ## 0.4.0
 
   - Better catalog selection
   - Treatment for no catalogs returned
-  - Splited asset_sync task in two other tasks: sync and asset_sync
-  - Configurable verify_ssl and timeout
+  - Splited asset_sync task in two other tasks: ```sync``` and ```asset_sync```
+  - Configurable ```verify_ssl``` and ```timeout```
   - Automatic connect/reconnect of commands
   - Bugfix: download command was not using the bucket/container name
   - Bugfix: the etag header must not be quoted

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nightcrawler_swift (0.4.0)
+    nightcrawler_swift (0.5.0)
       multi_mime (>= 1.0.1)
       rest-client
 
@@ -11,10 +11,14 @@ GEM
     byebug (3.2.0)
       columnize (~> 0.8)
       debugger-linecache (~> 1.2)
+    codeclimate-test-reporter (0.4.0)
+      simplecov (>= 0.7.1, < 1.0.0)
     columnize (0.8.9)
     debugger-linecache (1.2.0)
     diff-lcs (1.2.5)
+    docile (1.1.5)
     mime-types (2.3)
+    multi_json (1.10.1)
     multi_mime (1.0.1)
     netrc (0.7.7)
     rake (10.3.2)
@@ -33,6 +37,11 @@ GEM
     rspec-mocks (3.0.4)
       rspec-support (~> 3.0.0)
     rspec-support (3.0.4)
+    simplecov (0.9.0)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
 
 PLATFORMS
   ruby
@@ -40,6 +49,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.6)
   byebug
+  codeclimate-test-reporter
   nightcrawler_swift!
   rake
   rspec

data/README.md

@@ -6,18 +6,28 @@ Like the X-Men nightcrawler this gem teleports your assets to a OpenStack Swift
 
 Add this line to your application's Gemfile:
 
-    gem 'nightcrawler_swift'
+```ruby
+gem 'nightcrawler_swift'
+```
 
 And then execute:
 
-    $ bundle
+```sh
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install nightcrawler_swift
+```sh
+$ gem install nightcrawler_swift
+```
 
 ## Usage
 
+* [With Rails](https://github.com/tulios/nightcrawler_swift#with-rails)
+* [Programatically](https://github.com/tulios/nightcrawler_swift#programatically)
+* [Command Line](https://github.com/tulios/nightcrawler_swift#command-line)
+
 ### With Rails
 #### 1) Configure your swift credentials and options
 
@@ -37,20 +47,36 @@ __Optional configurations:__
 config.nightcrawler_swift.max_age = 3600 # default: nil
 config.nightcrawler_swift.verify_ssl = true # default: false
 config.nightcrawler_swift.timeout = 10 # in seconds, default: nil
+config.nightcrawler_swift.admin_url = "https://api.host.com/v1/AUTH_1234" # default: uses the admin_url returned by authentication
+config.nightcrawler_swift.public_url = "http://asset.host.com/v1/AUTH_1234" # default: uses the public_url returned by authentication
+config.nightcrawler_swift.retries = 3 # default: 5, to disable set it to false
+config.nightcrawler_swift.max_retry_time = 64 # in seconds, default: 30
 ```
 
-_max_age_ will be used to define *Cache-Control:max-age=<value>* header.
-
 By default it will use ```Rails.logger``` as logger, to change that use a different logger in configurations, like:
 
 ```ruby
 config.nightcrawler_swift.logger = Logger.new(STDOUT)
 ```
 
+##### further explanation of configurations
+
+> max_age
+
+It will be used to define *Cache-Control:max-age=<value>* header.
+
+> retries
+
+The number of times to retry the request before failing. To disable this feature set it to __false__.
+
+> max_retry_time
+
+Maximum delay in seconds between each retry. The delay will start with 1s and will double for each retry until this value.
+
 #### 2) Profit!
 
 ```sh
-rake nightcrawler_swift:rails:asset_sync
+$ rake nightcrawler_swift:rails:asset_sync
 ```
 
 It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container. To sync the public directory without the asset precompilation use the task: ```nightcrawler_swift:rails:sync```
@@ -74,7 +100,11 @@ __Optional configurations:__
 ```ruby
 max_age: 3600,
 verify_ssl: true,
-timeout: 10
+timeout: 10, # in seconds
+admin_url: "https://api.host.com/v1/AUTH_1234", # default: uses the admin_url returned by authentication
+public_url: "http://asset.host.com/v1/AUTH_1234", # default: uses the public_url returned by authentication
+retries: 3, # default: 5, to disable set it to false
+max_retry_time: 64 # in seconds, default: 30
 ```
 
 By default it will use ```Logger.new(STDOUT)``` as logger, to change that use:
@@ -89,6 +119,58 @@ NightcrawlerSwift.logger = Logger.new(StringIO.new)
 NightcrawlerSwift.sync File.expand_path("./my-dir")
 ```
 
+### Command Line
+
+The NightcrawlerSwift shell command (CLI) allows you to interact with your buckets/containers easily, it has the same commands of the gem. To see the help, use the cli without arguments or use the _-h_/_--help_ switch.
+
+```sh
+$ nswift # or nswift -h
+```
+
+```nswift``` will use the configurations stored at the file __.nswiftrc__ located at your home directory. If you try to use any command without the file, it will create a sample configuration for you, but you can create your own.
+
+The configuration is a __json__ file, named __.nswiftrc__. You can include any configuration available to the gem (see the other usages example to know each option available). Follow the format:
+
+```json
+{
+  "bucket": "<bucket/container name>",
+  "tenant_name": "<tenant name>",
+  "username": "<username>",
+  "password": "<password>",
+  "auth_url": "<auth url, ex: https://auth.url.com:123/v2.0/tokens>"
+}
+```
+
+The following commands are available through the cli:
+
+```sh
+$ nswift list
+```
+```sh
+$ nswift upload <real_path> <swift_path> # nswift upload robots.txt assets/robots.txt
+```
+```sh
+$ nswift download <swift_path> # nswift download assets/robots.txt > my-robots.txt
+```
+```sh
+$ nswift delete <swift_path> # nswift delete assets/robots.txt
+```
+```sh
+$ nswift url-for <swift_path> # nswift url-for assets/robots.txt
+```
+
+For any commands you could provide a different configuration file through the _-c_/_--config_ switch, as:
+
+```sh
+$ nswift list -c /dir/my-nswift-rc
+```
+
+and a different bucket/container name through the _-b_/_--bucket_ switch, as:
+
+```sh
+$ nwift list -b rogue
+```
+
 ## Commands
 
 NightcrawlerSwift has some useful built-in commands. All commands require the configuration and will __automatically__ connect/reconnect to keystone when necessary.
@@ -129,6 +211,13 @@ delete.execute "my_file_path.txt"
 # true / false
 ```
 
+### Sync
+
+```ruby
+sync = NightcrawlerSwift::Sync.new
+sync.execute "/dir/to/synchronize"
+```
+
 ## Connection
 
 To manually establish the connection with keystone, use:
@@ -145,6 +234,22 @@ NightcrawlerSwift.connection.connected?
 
 To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
 
+## Options
+
+After configure the NightcrawlerSwift you can access your configurations through the __options__ method, like:
+
+```ruby
+NightcrawlerSwift.options
+```
+
+The only difference is that you will access each configuration as a method instead of a hash style, like:
+
+```ruby
+NightcrawlerSwift.configure tenant_name: "rogue"
+
+# Can be used as:
+NightcrawlerSwift.options.tenant_name # "rogue"
+```
 
 ## Contributing
 

data/bin/nswift

@@ -5,4 +5,4 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
 require "nightcrawler_swift"
 require "nightcrawler_swift/cli"
 
-NightcrawlerSwift::CLI.new(ARGV).run
+NightcrawlerSwift::CLI::Runner.new(ARGV).run

data/lib/nightcrawler_swift/cli/commands/url_for.rb

@@ -0,0 +1,9 @@
+module NightcrawlerSwift::CLI
+  class UrlFor < NightcrawlerSwift::Command
+
+    def execute path
+      "#{connection.public_url}/#{options.bucket}/#{path}"
+    end
+
+  end
+end

data/lib/nightcrawler_swift/cli/formatters/basic.rb

@@ -0,0 +1,40 @@
+module NightcrawlerSwift::CLI
+  module Formatters
+    class Basic
+
+      def initialize runner
+        @runner = runner
+      end
+
+      def command_list command
+        array = command.new.execute
+        array.each {|hash| @runner.log hash["name"]}
+      end
+
+      def command_download command
+        filepath = @runner.argv.first
+        @runner.log command.new.execute(filepath)
+      end
+
+      def command_upload command
+        realpath = @runner.argv.shift
+        swiftpath = @runner.argv.shift
+
+        uploaded = command.new.execute swiftpath, File.open(File.expand_path(realpath), "r")
+        @runner.log(uploaded ? "success" : "failure")
+      end
+
+      def command_delete command
+        filepath = @runner.argv.first
+        deleted = command.new.execute(filepath).to_json
+        @runner.log(deleted ? "success" : "failure")
+      end
+
+      def command_url_for command
+        filepath = @runner.argv.first
+        @runner.log command.new.execute(filepath)
+      end
+
+    end
+  end
+end

data/lib/nightcrawler_swift/cli/opt_parser.rb

@@ -0,0 +1,81 @@
+module NightcrawlerSwift::CLI
+  class OptParser
+
+    attr_reader :parser
+
+    def initialize runner
+      @runner = runner
+      @parser = OptionParser.new
+      configure_instructions
+      configure_options
+    end
+
+    def parse!
+      @parser.parse!(@runner.argv)
+    end
+
+    def help
+      @parser.help
+    end
+
+    private
+    def configure_instructions
+      @parser.banner = "nswift #{NightcrawlerSwift::VERSION}"
+      @parser.separator "Usage: nswift command [options]"
+      @parser.separator ""
+      @parser.separator "commands:"
+      COMMANDS.keys.each do |key|
+        @parser.separator "    #{key}\t\t\t     #{COMMANDS[key][:description]}"
+      end
+
+      @parser.separator ""
+    end
+
+    def configure_options
+      @parser.separator "options:"
+      configure_option_bucket
+      configure_option_config
+      configure_option_help
+      configure_option_version
+    end
+
+    def configure_option_bucket
+      desc = "Alternative bucket/container name, overrides the '#{NightcrawlerSwift::CLI::CONFIG_FILE}' configuration"
+      @parser.on("-b", "--bucket=NAME", String, desc) do |name|
+        bucket = name.strip
+        @runner.options.bucket = bucket
+        @runner.log "Using bucket: #{@runner.options.bucket}"
+      end
+    end
+
+    def configure_option_config
+      @parser.on("-c", "--config=PATH", String, "Alternative '#{NightcrawlerSwift::CLI::CONFIG_FILE}' file") do |path|
+        path = File.expand_path(path.strip)
+
+        unless File.exist?(path)
+          @runner.log "Error: No such file or directory - #{path}"
+          exit 1
+        end
+
+        @runner.log "Using custom config file at: #{path}"
+        @runner.options.config_file = path
+        @runner.options.default_config_file = false
+      end
+    end
+
+    def configure_option_help
+      @parser.on_tail("-h", "--help", "Show this message") do
+        @runner.log help
+        exit
+      end
+    end
+
+    def configure_option_version
+      @parser.on_tail("-v", "--version", "Show version") do
+        @runner.log NightcrawlerSwift::VERSION
+        exit
+      end
+    end
+
+  end
+end

data/lib/nightcrawler_swift/cli/runner.rb

@@ -0,0 +1,127 @@
+module NightcrawlerSwift::CLI
+  class Runner
+    attr_reader :opt_parser, :options
+    attr_accessor :argv
+
+    def initialize argv
+      @argv = argv
+      configure_logger
+    end
+
+    def run
+      configure_default_options
+      configure_opt_parser
+      parse_parameters
+      @command_name = argv.shift
+      validate_command_and_options
+      execute_command if @command_name
+    end
+
+    def log string
+      NightcrawlerSwift.logger.info string
+    end
+
+    private
+    def configure_logger
+      STDOUT.sync = true
+      NightcrawlerSwift.logger.formatter = lambda {|severity, datetime, progname, msg| "#{msg}\n"}
+    end
+
+    def configure_default_options
+      user_home_dir = Dir.home
+      @options = OpenStruct.new
+      @options.configured = true
+      @options.default_config_file = true
+      @options.config_file = File.expand_path(File.join(user_home_dir, CONFIG_FILE))
+      @options.cache_file = File.expand_path(File.join(user_home_dir, CACHE_FILE))
+      @options.bucket = nil
+    end
+
+    def validate_command_and_options
+      if @command_name.nil? or argv.nil?
+        log @opt_parser.help
+        exit
+      end
+
+      unless COMMANDS.include?(@command_name)
+        log "Error: Unknown command '#{@command_name}'"
+        exit 1
+      end
+
+      unless options.configured
+        log "You must configure your swift credentials, take a look at:\n   #{options.config_file}"
+        exit 1
+      end
+    end
+
+    def config_hash
+      @config_hash ||= JSON.parse(File.read(options.config_file))
+    end
+
+    def execute_command
+      NightcrawlerSwift.configure config_hash
+      NightcrawlerSwift.options.bucket = @options.bucket if @options.bucket
+
+      connect_and_execute do
+        command_method = "command_#{command_name_normalized}"
+        command_class = COMMANDS[@command_name][:command]
+        Formatters::Basic.new(self).send(command_method, command_class)
+      end
+
+    rescue NightcrawlerSwift::Exceptions::BaseError, Errno::ENOENT => e
+      log "Error: #{e.message}"
+      exit 1
+    end
+
+    def command_name_normalized
+      @command_name.downcase.gsub(/-/, "_")
+    end
+
+    def connect_and_execute &block
+      path = options.cache_file
+      if File.exist?(path)
+        hash = JSON.parse File.read(path)
+        NightcrawlerSwift.connection.auth_response = OpenStruct.new(hash)
+        NightcrawlerSwift.connection.configure
+
+        token_id = NightcrawlerSwift.connection.token_id
+        NightcrawlerSwift.logger.debug "Cache found, restablishing connection with token_id: #{token_id}"
+      end
+
+      begin
+        block.call
+      ensure
+        File.open(path, "w") do |f|
+          f.write(NightcrawlerSwift.connection.auth_response.to_h.to_json)
+        end
+      end
+    end
+
+    def parse_parameters
+      @opt_parser.parse!
+      check_rcfile if options.default_config_file
+
+    rescue OptionParser::InvalidOption => e
+      log e.message
+      exit 1
+    end
+
+    def configure_opt_parser
+      @opt_parser = OptParser.new self
+    end
+
+    def check_rcfile
+      sample = NightcrawlerSwift::CLI.sample_rcfile
+      unless File.exist?(options.config_file)
+        File.open(options.config_file, "w") { |f|
+          f.write(sample)
+        }
+      end
+
+      if sample == File.read(options.config_file)
+        options.configured = false
+      end
+    end
+
+  end
+end