nightcrawler_swift 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8ef6b380623f8ce83cfea1491c10c44bfc8ed04
-  data.tar.gz: 9df2abbd80c6f1fceedf2dddc58d59be45e33520
+  metadata.gz: 206f894e8de465851f7a45dcea23615de0666dd4
+  data.tar.gz: 69f0ed04088e491d8d7bf1ea6e6b2c5c3bd03049
 SHA512:
-  metadata.gz: 8fdf97c07f42e76d5b83941b2f95d8cb32270bba360369c0b28547a25151ad56290aad0ea5ecfb076d1c012fb5abf4a331c579f216ed815dd342f9f5addf97d5
-  data.tar.gz: 9f2d5e258abc3f309cc57759a81938041ddf8ec49111276e3b4ee3bf6120edf527c875a01715b28a030aa970c508d5dbbfb185d702dd2d354e068cba11d6eff3
+  metadata.gz: 9432b08baae481fc88123911809c78ada1300653af4ecd1cadda04d5b3776bbdfdfb6288fd3284a42aacca7056632c1fd26f9bd1eb833401efdec43504eb9226
+  data.tar.gz: 1d14060fafd907d4e0185ff5612a84c087ea58234219638fae4f07eabd6d5c3d50a78b264d548d90c50e1731b60e58628a5e59267df9b8bd1ceb23536fb9b982

data/Changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.8.0
+
+  - Added support for ssl_version configuration
+  - Command for object metadata. Available for cli (closes #16, #17)
+
 ## 0.7.0
 
   - CLI now supports ```--no-cache``` which ignores the cached connection and removes the cache file if it exists (issue #12)

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    nightcrawler_swift (0.7.0)
+    nightcrawler_swift (0.8.0)
+      concurrent-ruby (~> 0.8.0)
       multi_mime (>= 1.0.1)
       rest-client
 
@@ -15,6 +16,8 @@ GEM
     codeclimate-test-reporter (0.4.0)
       simplecov (>= 0.7.1, < 1.0.0)
     columnize (0.9.0)
+    concurrent-ruby (0.8.0)
+      ref (~> 1.0, >= 1.0.5)
     debugger-linecache (1.2.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
@@ -23,6 +26,7 @@ GEM
     multi_mime (1.0.1)
     netrc (0.10.2)
     rake (10.3.2)
+    ref (1.0.5)
     rest-client (1.7.2)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)

data/README.md

@@ -74,6 +74,9 @@ config.nightcrawler_swift.ssl_client_key =
 
 # default: nil
 config.nightcrawler_swift.ssl_ca_file = "ca_certificate.pem"
+
+#default: nil
+config.nightcrawler_swift.ssl_version = "SSLv23"
 ```
 
 By default it will use ```Rails.logger``` as logger, to change that use a different logger in configurations, like:
@@ -212,6 +215,19 @@ $ nswift delete <swift_path> # nswift delete assets/robots.txt
 ```sh
 $ nswift url-for <swift_path> # nswift url-for assets/robots.txt
 ```
+```sh
+$ nswift metadata <swift_path>
+# {
+#   "date": "Tue, 31 Mar 2015 02:10:40 GMT",
+#   "content_type": "text/css",
+#   "content_length": "50013",
+#   "last_modified": "Wed, 15 Oct 2014 13:38:56 GMT",
+#   "etag": "e13839c545f32be23b942a41f3ea7724",
+#   "x_timestamp": "1413380335.38118",
+#   "cache_control": "public, max-age=604800",
+#   "x_trans_id": "tx824d632ef9e54685a7563-005519f838"
+# }
+```
 
 For any commands you could provide a different configuration file through the _-c_/_--config_ switch, as:
 
@@ -275,6 +291,37 @@ list.execute prefix: '/some/path'
 # [{"hash": "...", "name": "/some/path/with/my_file_path.txt"}, {}, {}, ...]
 ```
 
+### Metadata
+
+```ruby
+metadata = NightcrawlerSwift::Metadata.new
+metadata.execute "my_file.css"
+# {
+#   :date=>"Tue, 31 Mar 2015 01:46:10 GMT",
+#   :content_type=>"text/css",
+#   :content_length=>"50013",
+#   :last_modified=>"Wed, 15 Oct 2014 13:38:56 GMT",
+#   :etag=>"e13839c545f32be23b942a41f3ea7724",
+#   :x_timestamp=>"1413380335.38118",
+#   :cache_control=>"public, max-age=604800",
+#   :x_trans_id=>"tx92ca1becd1a74fa3ae1f2-005519fc62"
+# }
+
+# Consulting the bucket/container metadata
+metadata.execute
+# {
+#   :date=>"Tue, 31 Mar 2015 01:41:25 GMT",
+#   :content_type=>"application/xml; charset=utf-8",
+#   :content_length=>"98171",
+#   :x_container_object_count=>"374",
+#   :x_storage_policy=>"default-3x",
+#   :x_container_read=>".r:*",
+#   :x_container_bytes_used=>"7171468",
+#   :x_timestamp=>"1409750420.36261",
+#   :x_trans_id=>"tx17d8c66ab746436b8b7b6-005519fb40"
+# }
+```
+
 ### Delete
 
 ```ruby

data/lib/nightcrawler_swift/cli/formatters/basic.rb

@@ -30,6 +30,12 @@ module NightcrawlerSwift::CLI
         @runner.log(deleted ? "success" : "failure")
       end
 
+      def command_metadata command
+        filepath = @runner.argv.first
+        metadata = command.new.execute(filepath)
+        @runner.log JSON.pretty_generate(metadata)
+      end
+
       def command_url_for command
         filepath = @runner.argv.first
         @runner.log command.new.execute(filepath)

data/lib/nightcrawler_swift/cli.rb

@@ -32,6 +32,11 @@ module NightcrawlerSwift
         command: NightcrawlerSwift::Delete
       },
 
+      "metadata" => {
+        description: "Returns the object metadata in JSON format. Without <swift_path> returns the bucket/container metadata. Ex: nswift metadata assets/robots.txt",
+        command: NightcrawlerSwift::Metadata
+      },
+
       "url-for" => {
         description: "Returns the public url of an object. Format: nswift url-for <swift_path> Ex: nswift url-for robots.txt",
         command: NightcrawlerSwift::CLI::UrlFor

data/lib/nightcrawler_swift/command.rb

@@ -19,9 +19,18 @@ module NightcrawlerSwift
 
     protected
 
-    def get url, args = {}
-      prepare_args args
-      Gateway.new(url).request {|r| r.get headers_and_params(args)}
+    [
+      :get,
+      :head,
+      :delete
+    ].each do |http_verb|
+      define_method http_verb do |*method_args|
+        url = method_args[0]
+        args = method_args[1] || {}
+
+        prepare_args args
+        Gateway.new(url).request {|r| r.send(http_verb, headers_and_params(args))}
+      end
     end
 
     def put url, args = {}
@@ -29,11 +38,6 @@ module NightcrawlerSwift
       Gateway.new(url).request {|r| r.put args[:body], headers_and_params(args)}
     end
 
-    def delete url, args = {}
-      prepare_args args
-      Gateway.new(url).request {|r| r.delete headers_and_params(args)}
-    end
-
     private
     def prepare_args args
       args[:headers] ||= {}

data/lib/nightcrawler_swift/commands/metadata.rb

@@ -0,0 +1,13 @@
+module NightcrawlerSwift
+  class Metadata < Command
+
+    def execute path = nil
+      resource_url = "#{connection.internal_url}/#{options.bucket}/#{path}"
+      resource_url.gsub!(/\/$/, '') unless path
+
+      response = head resource_url
+      response.headers.symbolize_keys
+    end
+
+  end
+end

data/lib/nightcrawler_swift/commands/multithread_sync.rb

@@ -0,0 +1,44 @@
+module NightcrawlerSwift
+  class MultithreadSync < Command
+    DEFAULT_POOL_SIZE = 5
+
+    def initialize
+      require 'concurrent/utilities'
+      require 'concurrent/executors'
+      @logger = NightcrawlerSwift.logger
+    end
+
+    def execute args = {}
+      pool_size = args[:pool_size] || DEFAULT_POOL_SIZE
+      dir_path = args[:dir_path]
+
+      @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
+      @logger.info "[NightcrawlerSwift] multithread sync, #{Concurrent.processor_count} processors"
+
+      assets = Dir["#{dir_path}/**/**"].
+        reject {|fullpath| File.directory?(fullpath)}.
+        map {|fullpath|
+          path = fullpath.gsub("#{dir_path}/", "")
+          OpenStruct.new(path: path, fullpath: fullpath)
+        }
+
+      pool = Concurrent::FixedThreadPool.new pool_size
+
+      assets.each do |asset|
+        pool.post do
+          @logger.info "[NightcrawlerSwift] #{asset.path}"
+
+          upload = Upload.new
+          upload.execute asset.path, File.open(asset.fullpath, "r")
+        end
+      end
+
+      sleep(1) while pool.queue_length > 0
+
+      @logger.info "[NightcrawlerSwift] shutting down"
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
+  end
+end

data/lib/nightcrawler_swift/gateway.rb

@@ -59,7 +59,8 @@ module NightcrawlerSwift
         :verify_ssl,
         :ssl_client_cert,
         :ssl_client_key,
-        :ssl_ca_file
+        :ssl_ca_file,
+        :ssl_version
 
       ].inject({}) {|hash, key|
         hash.tap {

data/lib/nightcrawler_swift/version.rb

@@ -1,3 +1,3 @@
 module NightcrawlerSwift
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/nightcrawler_swift.rb

@@ -1,5 +1,6 @@
 require "cgi"
 require "date"
+require "json"
 require "logger"
 require "digest"
 require "ostruct"
@@ -16,9 +17,11 @@ require "nightcrawler_swift/connection"
 require "nightcrawler_swift/command"
 require "nightcrawler_swift/commands/upload"
 require "nightcrawler_swift/commands/download"
+require "nightcrawler_swift/commands/metadata"
 require "nightcrawler_swift/commands/list"
 require "nightcrawler_swift/commands/delete"
 require "nightcrawler_swift/commands/sync"
+require "nightcrawler_swift/commands/multithread_sync"
 require "nightcrawler_swift/railtie" if defined?(Rails)
 
 module NightcrawlerSwift

data/nightcrawler_swift.gemspec

@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rest-client"
-  spec.add_dependency "multi_mime",  ">= 1.0.1"
+  spec.add_dependency "multi_mime",      ">= 1.0.1"
+  spec.add_dependency "concurrent-ruby", "~> 0.8.0"
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"

data/spec/lib/nightcrawler_swift/cli/formatters/basic_spec.rb

@@ -100,6 +100,24 @@ describe NightcrawlerSwift::CLI::Formatters::Basic do
     end
   end
 
+  describe "#command_metadata" do
+    let :command_class do
+      NightcrawlerSwift::Metadata
+    end
+
+    let :metadata do
+      {some_metadata: true}
+    end
+
+    it "returns the metadata of the given path" do
+      expect(command_class).to receive(:new).and_return(command)
+      expect(command).to receive(:execute).with(filepath).and_return(metadata)
+      expect(runner).to receive(:log).with(JSON.pretty_generate(metadata))
+      expect(runner).to receive(:argv).and_return([filepath])
+      subject.command_metadata command_class
+    end
+  end
+
   describe "#command_url_for" do
     let :command_class do
       NightcrawlerSwift::CLI::UrlFor

data/spec/lib/nightcrawler_swift/command_spec.rb

@@ -15,7 +15,7 @@ describe NightcrawlerSwift::Command do
   end
 
   let(:connection) { NightcrawlerSwift::Connection.new }
-  let(:restclient) { double(:restclient, put: response, get: response, delete: response) }
+  let(:restclient) { double(:restclient, put: response, get: response, head: response, delete: response) }
   let(:response) { double(:response) }
   let(:url) { "http://url-com" }
   let(:token) { "token" }
@@ -28,6 +28,33 @@ describe NightcrawlerSwift::Command do
     end
   end
 
+  shared_examples "standard http method" do
+    let :execute_http do
+      subject.send http_verb, url, body: 'ignore_me', headers: { content_type: :json }, params: { b: 2 }
+    end
+
+    it_behaves_like "command with configured gateway"
+
+    it "ignores body" do
+      execute_http
+      expect(restclient).to have_received(http_verb).with(kind_of(Hash))
+    end
+
+    it "sends headers with token" do
+      execute_http
+      expect(restclient).to have_received(http_verb).with(hash_including({ content_type: :json, "X-Storage-Token" => token }))
+    end
+
+    it "sends params" do
+      execute_http
+      expect(restclient).to have_received(http_verb).with(hash_including(params: { b: 2 }))
+    end
+
+    it "returns RestClient response" do
+      expect(execute_http).to eql(response)
+    end
+  end
+
   describe "#connection" do
     context "when disconnected" do
       before do
@@ -59,57 +86,18 @@ describe NightcrawlerSwift::Command do
   end
 
   describe "#get" do
-    let :execute_http do
-      subject.send :get, url, body: 'ignore_me', headers: { content_type: :json }, params: { b: 2 }
-    end
-
-    it_behaves_like "command with configured gateway"
-
-    it "ignores body" do
-      execute_http
-      expect(restclient).to have_received(:get).with(kind_of(Hash))
-    end
-
-    it "sends headers with token" do
-      execute_http
-      expect(restclient).to have_received(:get).with(hash_including({ content_type: :json, "X-Storage-Token" => token }))
-    end
-
-    it "sends params" do
-      execute_http
-      expect(restclient).to have_received(:get).with(hash_including(params: { b: 2 }))
-    end
+    let(:http_verb) { :get }
+    it_behaves_like "standard http method"
+  end
 
-    it "returns RestClient response" do
-      expect(execute_http).to eql(response)
-    end
+  describe "#head" do
+    let(:http_verb) { :head }
+    it_behaves_like "standard http method"
   end
 
   describe "#delete" do
-    let :execute_http do
-      subject.send :delete, url, body: 'ignore_me', headers: { content_type: :json }, params: { b: 2 }
-    end
-
-    it_behaves_like "command with configured gateway"
-
-    it "ignores body" do
-      execute_http
-      expect(restclient).to have_received(:delete).with(kind_of(Hash))
-    end
-
-    it "sends headers with token" do
-      execute_http
-      expect(restclient).to have_received(:delete).with(hash_including({ content_type: :json, "X-Storage-Token" => token }))
-    end
-
-    it "sends params" do
-      execute_http
-      expect(restclient).to have_received(:delete).with(hash_including(params: { b: 2 }))
-    end
-
-    it "returns RestClient response" do
-      expect(execute_http).to eql(response)
-    end
+    let(:http_verb) { :delete }
+    it_behaves_like "standard http method"
   end
 
   describe "#put" do

data/spec/lib/nightcrawler_swift/commands/metadata_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe NightcrawlerSwift::Metadata do
+
+  let(:connection) { NightcrawlerSwift::Connection.new }
+  let(:token) { "token" }
+  let(:expires_at) { (DateTime.now + 60).to_time }
+  let(:internal_url) { "server-url" }
+  let(:bucket) { "rogue" }
+
+  subject do
+    NightcrawlerSwift::Metadata.new
+  end
+
+  before do
+    allow(NightcrawlerSwift).to receive(:connection).and_return(connection)
+    allow(connection).to receive(:token_id).and_return(token)
+    allow(connection).to receive(:expires_at).and_return(expires_at)
+    allow(connection).to receive(:internal_url).and_return(internal_url)
+    NightcrawlerSwift.configure bucket: bucket
+  end
+
+  describe "#execute" do
+    let(:file_path) { "file_path.css" }
+    let(:execute) { subject.execute file_path }
+
+    let :response do
+      double(:response, code: 200, body: "", headers: {
+        "content_type" => "text/css",
+        "content_length" => "50013",
+        "last_modified" => "Wed, 15 Oct 2014 13:38:56 GMT",
+        "etag" => "e13839c545f32be23b942a41f3ea7724",
+        "x_timestamp" => "1413380335.38118",
+        "cache_control" => "public, max-age=604800",
+        "x_trans_id" => "tx824d632ef9e54685a7563-005519f838"
+      })
+    end
+
+    before do
+      allow(subject).to receive(:head).and_return(response)
+    end
+
+    context "success" do
+      it "gets using internal url" do
+        execute
+        expect(subject).to have_received(:head).with("server-url/#{bucket}/#{file_path}")
+      end
+
+      it "returns headers with symbolized keys" do
+        output = execute
+        expect(output).to eql response.headers.symbolize_keys
+      end
+    end
+
+    context "when the path was not informed" do
+      let(:file_path) { nil }
+
+      it "removes the trailing slash" do
+        execute
+        expect(subject).to have_received(:head).with("server-url/#{bucket}")
+      end
+    end
+  end
+end

data/spec/lib/nightcrawler_swift/gateway_spec.rb

@@ -81,6 +81,22 @@ describe NightcrawlerSwift::Gateway do
         end
       end
 
+      context "but only 'ssl_version'" do
+        let :opts do
+          {ssl_version: 'sslversion'}
+        end
+
+        it "uses the configured option" do
+          expect(RestClient::Resource).to receive(:new).with(
+            url,
+            timeout: NightcrawlerSwift.options.timeout,
+            verify_ssl: NightcrawlerSwift.options.verify_ssl,
+            ssl_version: NightcrawlerSwift.options.ssl_version
+          )
+          subject
+        end
+      end
+
       context "using all options" do
         let :opts do
           {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nightcrawler_swift
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - tulios
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-12 00:00:00.000000000 Z
+date: 2015-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -141,6 +155,8 @@ files:
 - lib/nightcrawler_swift/commands/delete.rb
 - lib/nightcrawler_swift/commands/download.rb
 - lib/nightcrawler_swift/commands/list.rb
+- lib/nightcrawler_swift/commands/metadata.rb
+- lib/nightcrawler_swift/commands/multithread_sync.rb
 - lib/nightcrawler_swift/commands/sync.rb
 - lib/nightcrawler_swift/commands/upload.rb
 - lib/nightcrawler_swift/connection.rb
@@ -168,6 +184,7 @@ files:
 - spec/lib/nightcrawler_swift/commands/delete_spec.rb
 - spec/lib/nightcrawler_swift/commands/download_spec.rb
 - spec/lib/nightcrawler_swift/commands/list_spec.rb
+- spec/lib/nightcrawler_swift/commands/metadata_spec.rb
 - spec/lib/nightcrawler_swift/commands/sync_spec.rb
 - spec/lib/nightcrawler_swift/commands/upload_spec.rb
 - spec/lib/nightcrawler_swift/connection_spec.rb
@@ -219,6 +236,7 @@ test_files:
 - spec/lib/nightcrawler_swift/commands/delete_spec.rb
 - spec/lib/nightcrawler_swift/commands/download_spec.rb
 - spec/lib/nightcrawler_swift/commands/list_spec.rb
+- spec/lib/nightcrawler_swift/commands/metadata_spec.rb
 - spec/lib/nightcrawler_swift/commands/sync_spec.rb
 - spec/lib/nightcrawler_swift/commands/upload_spec.rb
 - spec/lib/nightcrawler_swift/connection_spec.rb