nightcrawler_swift 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 792e1b2a4c65b85291d7a5c733fbd238f6b4f531
-  data.tar.gz: b34fa251f341f191b7627d8ac200bb0daa43fdbd
+  metadata.gz: 616e1624837906ed1b5381823a1de196ba8b41e7
+  data.tar.gz: 2e4d0b8eb39afb34e6cda6d5e7725c6938a3a817
 SHA512:
-  metadata.gz: be4f048df14fe2453b4fc22ed8e495434c5f4fb51009f8e5765fae6f61cbc160dcf12ae4c2d274e4aa69096884b629d8e83538f03bd14dd4992ad39dc1e66f91
-  data.tar.gz: d9bd21e55c32c1ee73f16e817c7edf2c529552d94402c615a63a37026f01eb5ba477b21c461878f35fcd54953e574553dda79e5ce6f50cf5b745e61ad55030a7
+  metadata.gz: 193eccd937c649b468e4b24cff3728d22a6acd7de8ac824876732bff14fbb145e33dafb222ff6069e350912c849b7c55d5e75a56e25a11357e6a8e9d368ece6c
+  data.tar.gz: 466c00ebec3b67d64efca29ec820afea48ad74a0e702617dc91e570cf0c8c08bfca7a0808b2d8c8d01d3c09d398c64d7df7e7766312e1c35826fa500dd1af264

data/Changelog.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.4.0
+
+  - Better catalog selection
+  - Treatment for no catalogs returned
+  - Splited asset_sync task in two other tasks: sync and asset_sync
+  - Configurable verify_ssl and timeout
+  - Automatic connect/reconnect of commands
+  - Bugfix: download command was not using the bucket/container name
+  - Bugfix: the etag header must not be quoted
+  - CLI with basic commands (list, download, upload and delete)
+
 ## 0.3.0
 
   - Included ```Cache-Control``` header to upload command through max_age configuration parameter

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nightcrawler_swift (0.3.0)
+    nightcrawler_swift (0.4.0)
       multi_mime (>= 1.0.1)
       rest-client
 

data/README.md

@@ -1,7 +1,4 @@
-[![Code Climate](https://codeclimate.com/github/tulios/nightcrawler_swift/badges/gpa.svg)](https://codeclimate.com/github/tulios/nightcrawler_swift)
-[![Travis](https://api.travis-ci.org/tulios/nightcrawler_swift.svg?branch=master)](https://travis-ci.org/tulios/nightcrawler_swift)
-[![Gem Version](https://badge.fury.io/rb/nightcrawler_swift.svg)](http://badge.fury.io/rb/nightcrawler_swift)
-# Nightcrawler Swift
+# Nightcrawler Swift [![Code Climate](https://codeclimate.com/github/tulios/nightcrawler_swift/badges/gpa.svg)](https://codeclimate.com/github/tulios/nightcrawler_swift) [![Travis](https://api.travis-ci.org/tulios/nightcrawler_swift.svg?branch=master)](https://travis-ci.org/tulios/nightcrawler_swift) [![Gem Version](https://badge.fury.io/rb/nightcrawler_swift.svg)](http://badge.fury.io/rb/nightcrawler_swift)
 
 Like the X-Men nightcrawler this gem teleports your assets to a OpenStack Swift bucket/container. It was designed to sync your assets with OpenStack Swift and allow some operations with your buckets/containers.
 
@@ -32,9 +29,17 @@ config.nightcrawler_swift.tenant_name = "nightcrawler"
 config.nightcrawler_swift.username = "my_username1"
 config.nightcrawler_swift.password = "my_password1"
 config.nightcrawler_swift.auth_url = "https://auth.url.com:123/v2.0/tokens"
-config.nightcrawler_swift.max_age = 3600 #optional
 ```
-**max_age** will be used to define *Cache-Control:max-age=<value>* header. It's not required.
+
+__Optional configurations:__
+
+```ruby
+config.nightcrawler_swift.max_age = 3600 # default: nil
+config.nightcrawler_swift.verify_ssl = true # default: false
+config.nightcrawler_swift.timeout = 10 # in seconds, default: nil
+```
+
+_max_age_ will be used to define *Cache-Control:max-age=<value>* header.
 
 By default it will use ```Rails.logger``` as logger, to change that use a different logger in configurations, like:
 
@@ -48,7 +53,7 @@ config.nightcrawler_swift.logger = Logger.new(STDOUT)
 rake nightcrawler_swift:rails:asset_sync
 ```
 
-It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container.
+It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container. To sync the public directory without the asset precompilation use the task: ```nightcrawler_swift:rails:sync```
 
 ### Programatically
 
@@ -57,14 +62,21 @@ It will invoke ```rake assets:precompile``` and will copy your public directory
 ```ruby
 NightcrawlerSwift.configure({
   bucket: "rogue",
-  tenant_name: "nightcrawler"
+  tenant_name: "nightcrawler",
   username: "my_username1",
   password: "my_password1",
-  auth_url: "https://auth.url.com:123/v2.0/tokens",
-  max_age: 3600 #optional
+  auth_url: "https://auth.url.com:123/v2.0/tokens"
 })
 ```
 
+__Optional configurations:__
+
+```ruby
+max_age: 3600,
+verify_ssl: true,
+timeout: 10
+```
+
 By default it will use ```Logger.new(STDOUT)``` as logger, to change that use:
 
 ```ruby
@@ -79,21 +91,7 @@ NightcrawlerSwift.sync File.expand_path("./my-dir")
 
 ## Commands
 
-NightcrawlerSwift has some useful built-in commands. All commands require the configuration and a established connection.
-
-To Establish the connection, use:
-
-```ruby
-NightcrawlerSwift.connection.connect!
-```
-
-To check if the connection is still valid, use:
-
-```ruby
-NightcrawlerSwift.connection.connected?
-```
-
-To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
+NightcrawlerSwift has some useful built-in commands. All commands require the configuration and will __automatically__ connect/reconnect to keystone when necessary.
 
 ### Upload
 
@@ -131,6 +129,23 @@ delete.execute "my_file_path.txt"
 # true / false
 ```
 
+## Connection
+
+To manually establish the connection with keystone, use:
+
+```ruby
+NightcrawlerSwift.connection.connect!
+```
+
+To check if the connection is still valid, use:
+
+```ruby
+NightcrawlerSwift.connection.connected?
+```
+
+To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
+
+
 ## Contributing
 
 1. Fork it ( https://github.com/tulios/nightcrawler_swift/fork )

data/bin/nswift

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
+
+require "nightcrawler_swift"
+require "nightcrawler_swift/cli"
+
+NightcrawlerSwift::CLI.new(ARGV).run

data/lib/nightcrawler_swift/cli.rb

@@ -0,0 +1,208 @@
+require "optparse"
+require "json"
+
+module NightcrawlerSwift
+  class CLI
+    CONFIG_FILE = ".nswiftrc"
+    CACHE_FILE = ".nswift_cache"
+    COMMANDS = {
+      "list" => {
+        description: "Lists all files of the bucket/container. Ex: nswift list",
+        command: NightcrawlerSwift::List
+      },
+
+      "download" => {
+        description: "Downloads a file by path. Format: nswift download <swift path> Ex: nswift download assets/robots.txt > my-robots.txt",
+        command: NightcrawlerSwift::Download
+      },
+
+      "upload" => {
+        description: "Uploads some file. Format: nswift upload <real path> <swift path> Ex: nswift upload assets/robots.txt robots.txt",
+        command: NightcrawlerSwift::Upload
+      },
+
+      "delete" => {
+        description: "Deletes a file by path. Format: nswift delete <swift path> Ex: nswift delete robots.txt",
+        command: NightcrawlerSwift::Delete
+      }
+    }
+
+    attr_reader :opt_parser, :options
+    attr_accessor :argv
+
+    def initialize argv
+      @argv = argv
+      STDOUT.sync = true
+      NightcrawlerSwift.logger.formatter = lambda {|severity, datetime, progname, msg| "#{msg}\n"}
+    end
+
+    def run
+      configure_default_options
+      parse_parameters
+      @command_name = argv.shift
+      validate_command_and_options
+      execute_command if @command_name
+    end
+
+    protected
+    def command_list command
+      array = command.new.execute
+      array.each {|hash| log hash["name"]}
+    end
+
+    def command_download command
+      filepath = argv.first
+      log command.new.execute(filepath)
+    end
+
+    def command_upload command
+      realpath = argv.shift
+      swiftpath = argv.shift
+
+      uploaded = command.new.execute swiftpath, File.open(File.expand_path(realpath), "r")
+      log(uploaded ? "success" : "failure")
+    end
+
+    def command_delete command
+      filepath = argv.first
+      deleted = command.new.execute(filepath).to_json
+      log(deleted ? "success" : "failure")
+    end
+
+    def user_home_dir
+      Dir.home
+    end
+
+    def log string
+      NightcrawlerSwift.logger.info string
+    end
+
+    private
+    def configure_default_options
+      @options = OpenStruct.new
+      @options.configured = true
+      @options.default_config_file = true
+      @options.config_file = File.expand_path(File.join(user_home_dir, CONFIG_FILE))
+      @options.cache_file = File.expand_path(File.join(user_home_dir, CACHE_FILE))
+      @options.command = nil
+    end
+
+    def validate_command_and_options
+      if @command_name.nil? or argv.nil?
+        log opt_parser.help
+        exit
+      end
+
+      unless options.configured
+        log "You must configure your swift credentials, take a look at:\n   #{options.config_file}"
+        exit
+      end
+    end
+
+    def config_hash
+      @config_hash ||= JSON.parse(File.read(options.config_file))
+
+    rescue Errno::ENOENT => e
+      log "No such file or directory - #{options.config_file}"
+      exit 1
+    end
+
+    def execute_command
+      NightcrawlerSwift.configure config_hash
+      connect_and_execute do
+        if command = COMMANDS[@command_name]
+          command_module = command[:command]
+          command_method = "command_#{@command_name}"
+          self.send(command_method, command_module)
+        end
+      end
+
+    rescue Exceptions::BaseError => e
+      log e.message
+      exit 1
+    end
+
+    def connect_and_execute &block
+      path = options.cache_file
+      if File.exist?(path)
+        hash = JSON.parse File.read(path)
+        NightcrawlerSwift.connection.auth_response = OpenStruct.new(hash)
+        NightcrawlerSwift.connection.configure
+      end
+
+      begin
+        block.call
+      ensure
+        File.open(path, "w") do |f|
+          f.write(NightcrawlerSwift.connection.auth_response.to_h.to_json)
+        end
+      end
+    end
+
+    def parse_parameters
+      configure_opt_parser
+      opt_parser.parse!(argv)
+      check_rcfile if options.default_config_file
+
+    rescue OptionParser::InvalidOption => e
+      log e.message
+      exit 1
+    end
+
+    def configure_opt_parser
+      @opt_parser = OptionParser.new do |opts|
+        opts.banner = "nswift #{NightcrawlerSwift::VERSION}"
+        opts.separator "Usage: nswift command [options]"
+
+        opts.separator ""
+        opts.separator "commands:"
+        COMMANDS.keys.each do |key|
+          opts.separator "    #{key}\t\t\t     #{COMMANDS[key][:description]}"
+        end
+
+        opts.separator ""
+        opts.separator "options:"
+
+        opts.on("-c", "--config=PATH", String, "Alternative '#{CONFIG_FILE}' file") do |path|
+          path = File.expand_path(path.strip)
+          log "Using custom config file at: #{path}"
+          options.config_file = path
+          options.default_config_file = false
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          log opts.help
+          exit
+        end
+
+        opts.on_tail("-v", "--version", "Show version") do
+          log NightcrawlerSwift::VERSION
+          exit
+        end
+      end
+    end
+
+    def check_rcfile
+      unless File.exist?(options.config_file)
+        File.open(options.config_file, "w") { |f|
+          f.write(sample_rcfile)
+        }
+      end
+
+      if sample_rcfile == File.read(options.config_file)
+        options.configured = false
+      end
+    end
+
+    def sample_rcfile
+      JSON.pretty_generate({
+        bucket: "<bucket/container name>",
+        tenant_name: "<tenant name>",
+        username: "<username>",
+        password: "<password>",
+        auth_url: "<auth url, ex: https://auth.url.com:123/v2.0/tokens>"
+      })
+    end
+
+  end
+end

data/lib/nightcrawler_swift/command.rb

@@ -2,7 +2,13 @@ module NightcrawlerSwift
   class Command
 
     def connection
-      NightcrawlerSwift.connection
+      NightcrawlerSwift.connection.tap do |conn|
+        conn.connect! unless conn.connected?
+      end
+    end
+
+    def options
+      NightcrawlerSwift.options
     end
 
     def execute
@@ -32,7 +38,11 @@ module NightcrawlerSwift
     private
 
     def resource_for url
-      RestClient::Resource.new url, verify_ssl: false
+      RestClient::Resource.new(
+        url,
+        verify_ssl: options.verify_ssl,
+        timeout: options.timeout
+      )
     end
 
     def prepare_params params

data/lib/nightcrawler_swift/{delete.rb → commands/delete.rb}

@@ -3,7 +3,7 @@ module NightcrawlerSwift
 
     def execute path
       response = delete "#{connection.upload_url}/#{path}", headers: {accept: :json }
-      JSON.parse(response.body)
+      [200, 201].include?(response.code)
 
     rescue RestClient::ResourceNotFound => e
       raise Exceptions::NotFoundError.new(e)

data/lib/nightcrawler_swift/{download.rb → commands/download.rb}

@@ -2,7 +2,7 @@ module NightcrawlerSwift
   class Download < Command
 
     def execute path
-      response = get "#{connection.public_url}/#{path}"
+      response = get "#{connection.public_url}/#{options.bucket}/#{path}"
       response.body
 
     rescue RestClient::ResourceNotFound => e

data/lib/nightcrawler_swift/{sync.rb → commands/sync.rb}

@@ -7,12 +7,12 @@ module NightcrawlerSwift
     end
 
     def execute dir_path
-      @logger.info "dir_path: #{dir_path}"
+      @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
       Dir["#{dir_path}/**/**"].each do |fullpath|
         path = fullpath.gsub("#{dir_path}/", "")
 
         unless File.directory?(fullpath)
-          @logger.info path
+          @logger.info "[NightcrawlerSwift] #{path}"
           @upload.execute path, File.open(fullpath, "r")
         end
       end

data/lib/nightcrawler_swift/commands/upload.rb

@@ -0,0 +1,25 @@
+module NightcrawlerSwift
+  class Upload < Command
+
+    def execute path, file
+      content = file.read
+      headers = {etag: etag(content), content_type: content_type(file)}
+      headers.merge!(cache_control: "max-age=#{options.max_age}") if options.max_age
+      response = put "#{connection.upload_url}/#{path}", body: content, headers: headers
+      [200, 201].include?(response.code)
+
+    rescue RestClient::UnprocessableEntity => e
+      raise Exceptions::ValidationError.new(e)
+    end
+
+    private
+    def content_type file
+      MultiMime.by_file(file).content_type
+    end
+
+    def etag content
+      Digest::MD5.hexdigest(content)
+    end
+
+  end
+end