openstack-swift 0.0.1 → 0.1.0

data/Gemfile

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source "http://gems.locaweb.com.br"
 
 # Specify your gem's dependencies in openstack-swift.gemspec
 gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    openstack-swift (0.1.0)
+
+GEM
+  remote: http://gems.locaweb.com.br/
+  specs:
+    archive-tar-minitar (0.5.2)
+    columnize (0.3.4)
+    crack (0.1.8)
+    diff-lcs (1.1.2)
+    httparty (0.7.8)
+      crack (= 0.1.8)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  httparty
+  openstack-swift!
+  rspec (~> 2.6)
+  ruby-debug19

data/config/swift.yml

@@ -0,0 +1,4 @@
+:swift:
+  :url: "https://sw:8080/auth/v1.0"
+  :user: "system:root"
+  :pass: "testpass"

data/lib/openstack-swift.rb

@@ -1,5 +1,41 @@
+# -*- coding: UTF-8 -*-
+require "httparty"
+require "net/http"
+require "net/https"
+require "fileutils"
+require "openstack-swift/api"
+require "openstack-swift/errors"
+require "openstack-swift/client"
+require "openstack-swift/swift_config"
+
 module Openstack
   module Swift
-    # Your code goes here...
+  end
+end
+
+module Net
+  class HTTPGenericRequest
+    private
+    def send_request_with_body_stream(sock, ver, path, f)
+      unless content_length() or chunked?
+        raise ArgumentError,
+          "Content-Length not given and Transfer-Encoding is not `chunked'"
+      end
+      supply_default_content_type
+      write_header sock, ver, path
+      if chunked?
+        while s = f.read(1024)
+          sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+        end
+        sock.write "0\r\n\r\n"
+      else
+        bytes_written = 0
+        buffer=1024
+        while (s = f.read(buffer)) && bytes_written < content_length()
+          sock.write s
+          bytes_written += buffer
+        end
+      end
+    end
   end
 end

data/lib/openstack-swift/api.rb

@@ -0,0 +1,147 @@
+# -*- coding: UTF-8 -*-
+module Openstack
+  module Swift
+    module Api
+      extend self
+
+      # Authentication method to get the url and token to conect to swift
+      # Returns:
+      #   x-storage-url
+      #   x-storage-token
+      #   x-auth-token
+      def auth(proxy, user, password)
+        res = HTTParty.get(proxy, :headers => {"X-Auth-User" => user, "X-Auth-Key" => password})
+        raise AuthenticationError unless res.code == 200
+
+        [res.headers["x-storage-url"],res.headers["x-storage-token"],res.headers["x-auth-token"]]
+      end
+
+      # Get informations about the currect account used to connect to swift
+      # Returns:
+      #   x-account-bytes-used
+      #   x-account-object-count
+      #   x-account-container-count
+      def account(url, token)
+        query = {:format => "json"}
+        HTTParty.head(url, :headers => {"X-Auth-Token"=> token}, :query => query).headers
+      end
+
+      # List containers
+      # Note that swift only returns 1000 items, so to list more than this
+      # you should use the marker option as the name of the last returned item (1000th item)
+      # to return the next sequency (1001 - 2000)
+      # query options: marker, prefix, limit
+      def containers(url, token, query = {})
+        query = query.merge(:format => "json")
+        res = HTTParty.get(url, :headers => {"X-Auth-Token"=> token}, :query => query)
+        res.to_a
+      end
+
+      # Get all objects for a given container
+      # Query options:
+      #   marker
+      #   prefix
+      #   limit
+      #   delimiter
+      def objects(url, token, container, query = {})
+        query = query.merge(:format => "json")
+        res = HTTParty.get("#{url}/#{container}", :headers => {"X-Auth-Token"=> token}, :query => query)
+        res.to_a
+      end
+
+      # Delete a container for a given name from swift
+      def delete_container(url, token, container)
+        res = HTTParty.delete("#{url}/#{container}", :headers => {"X-Auth-Token"=> token})
+        raise "Could not delete container '#{container}'" if res.code < 200 or res.code >= 300
+        true
+      end
+
+      # Create a container on swift from a given name
+      def create_container(url, token, container)
+        res = HTTParty.put("#{url}/#{container}", :headers => {"X-Auth-Token"=> token})
+        raise "Could not create container '#{container}'" if res.code < 200 or res.code >= 300
+        true
+      end
+
+      # Get the object stat given the object name and the container this object is in
+      def object_stat(url, token, container, object)
+        url = "#{url}/#{container}/#{object}"
+        query = {:format => "json"}
+        HTTParty.head(url, :headers => {"X-Auth-Token"=> token}, :query => query).headers
+      end
+
+      # Creates the manifest file for a splitted upload
+      # Given the container and file path a manifest is created to guide the downloads of this
+      # splitted file
+      def create_manifest(url, token, container, file_path)
+        file_name = file_path.match(/.+\/(.+?)$/)[1]
+        file_size  = File.size(file_path)
+        file_mtime = File.mtime(file_path).to_f.round(2)
+        manifest_path = "#{container}_segments/#{file_name}/#{file_mtime}/#{file_size}/"
+
+        res = HTTParty.put("#{url}/#{container}/#{file_name}", :headers => {
+          "X-Auth-Token" => token,
+          "x-object-manifest" => manifest_path,
+          "Content-Type" => "application/octet-stream",
+          "Content-Length" => "0"
+        })
+
+        raise "Could not create manifest for '#{file_path}'" if res.code < 200 or res.code >= 300
+        true
+      end
+
+      # Downloads an object (file) to disk and returns the saved file path
+      def download_object(url, token, container, object, file_name=nil)
+        file_name ||= "/tmp/swift/#{container}/#{object}"
+
+        # creating directory if it doesn't exist
+        FileUtils.mkdir_p(file_name.match(/(.+)\/.+?$/)[1])
+        file = File.open(file_name, "wb")
+        uri = URI.parse("#{url}/#{container}/#{object}")
+
+        req = Net::HTTP::Get.new(uri.path)
+        req.add_field("X-Auth-Token", token)
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        md5 = Digest::MD5.new
+
+        http.request(req) do |res|
+          res.read_body do |chunk|
+            file.write chunk
+            md5.update(chunk)
+          end
+
+          raise "MD5 checksum failed for #{container}/#{object}" if res["x-object-manifest"].nil? && res["etag"] != md5.hexdigest
+        end
+
+        file_name
+      ensure
+        file.close rescue nil
+      end
+
+      # Uploads a given object to a given container
+      def upload_object(url, token, container, file_path, options={})
+        options[:object_name] ||= file_path.match(/.+\/(.+?)$/)[1]
+        file = File.open(file_path, "rb")
+
+        file.seek(options[:position]) if options[:position]
+        uri = URI.parse("#{url}/#{container}/#{options[:object_name]}")
+
+        req = Net::HTTP::Put.new(uri.path)
+        req.add_field("X-Auth-Token", token)
+        req.body_stream = file
+        req.content_length = options[:size] || File.size(file_path)
+        req.content_type = "application/octet-stream"
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.request(req)
+      ensure
+        file.close rescue nil
+      end
+    end
+  end
+end

data/lib/openstack-swift/client.rb

@@ -0,0 +1,121 @@
+# -*- coding: UTF-8 -*-
+module Openstack
+  module Swift
+    class Client
+      SWIFT_API = Openstack::Swift::Api
+      MAX_SIZE = 4 * 1024 ** 3
+
+      # Initialize method
+      # It uses the authenticate method to store the tokens for future requests
+      def initialize(proxy, user, password)
+        @proxy, @user, @password = proxy, user, password
+        authenticate!
+      end
+
+      # Authentication method
+      # It stores the authentication url and token for future commands
+      # avoiding to request a new token for each request
+      # It should be used to force a new token
+      def authenticate!
+        @url, _, @token = SWIFT_API.auth(@proxy, @user, @password)
+
+        if @url.blank? or @token.blank?
+          raise AuthenticationError
+        else
+          true
+        end
+      end
+
+      # Returns the following informations about the object:
+      #   bytes_used: Number of bytes used by this account
+      #   object_count: Number of objects that this account have allocated
+      #   container_count: Number of container
+      def account_info
+        headers = SWIFT_API.account(@url, @token)
+        {
+          "bytes_used" => headers["x-account-bytes-used"],
+          "object_count" => headers["x-account-object-count"],
+          "container_count" => headers["x-account-container-count"]
+        }
+      end
+
+      # Returns the following informations about the account:
+      #   last_modified
+      #   md5
+      #   content_type
+      #   manifest
+      #   content_length
+      def object_info(container, object)
+        headers = SWIFT_API.object_stat(@url, @token, container, object)
+        {
+          "last_modified" => headers["last-modified"],
+          "md5" => headers["etag"],
+          "content_type" => headers["content-type"],
+          "manifest" => headers["x-object-manifest"],
+          "content_length" => headers["content-length"]
+        }
+      end
+
+      # This method uploads a file from a given to a given container
+      def upload(container, file_path, options={})
+        options[:segments_size] ||= MAX_SIZE
+
+        SWIFT_API.create_container(@url, @token, container) rescue nil
+
+        file_name, file_mtime, file_size  = file_info(file_path)
+
+        if file_size > options[:segments_size]
+          SWIFT_API.create_container(@url, @token, "#{container}_segments") rescue nil
+
+          segments_minus_one = file_size / options[:segments_size]
+          last_piece = file_size - segments_minus_one * options[:segments_size]
+          segments_minus_one.times do |segment|
+            upload_path_for(file_path, segment)
+            SWIFT_API.upload_object(
+              @url, @token, "#{container}_segments", file_path,
+              :size => options[:segments_size],
+              :position => options[:segments_size] * segment,
+              :object_name => segment_path
+            )
+          end
+
+          SWIFT_API.upload_object(
+            @url, @token, "#{container}_segments", file_path,
+            :size => last_piece,
+            :position => options[:segments_size] * segments_minus_one,
+            :object_name => upload_path_for(file_path, segments_minus_one)
+          )
+
+          SWIFT_API.create_manifest(@url, @token, container, file_path)
+        else
+          SWIFT_API.upload_object(@url, @token, container, file_path)
+        end
+      end
+
+      # This method downloads a object from a given container
+      def download(container, object)
+        SWIFT_API.download_object(@url, @token, container, object)
+      end
+
+     private
+
+      # Returns the standard swift path for a given file path and segment
+      def upload_path_for(file_path, segment)
+        "%s/%s/s/%08d" % (file_info(file_path) << segment)
+      end
+
+      # Get relevant informations about a file
+      # Returns an array with:
+      #   file_name
+      #   file_mtime
+      #   file_size
+      def file_info(file_path)
+        [
+          file_path.match(/.+\/(.+?)$/)[1],
+          File.mtime(file_path).to_f.round(2),
+          File.size(file_path)
+        ]
+      end
+    end
+  end
+end

data/lib/openstack-swift/errors.rb

@@ -0,0 +1,4 @@
+# -*- coding: UTF-8 -*-
+module Openstack::Swift
+  class AuthenticationError < Exception; end
+end

data/lib/openstack-swift/swift_config.rb

@@ -0,0 +1,19 @@
+# -*- coding: UTF-8 -*-
+module Openstack
+  module SwiftConfig
+    extend self
+
+    def [](name)
+      configs[name]
+    end
+
+    module_function
+    def configs
+      @config_file ||= load_file
+    end
+
+    def load_file
+      YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../../config/swift.yml")[:swift]
+    end
+  end
+end

data/lib/openstack-swift/version.rb

@@ -1,5 +1,6 @@
+# -*- coding: UTF-8 -*-
 module Openstack
   module Swift
-    VERSION = "0.0.1"
+    VERSION = "0.1.0"
   end
 end

data/openstack-swift.gemspec

@@ -8,14 +8,16 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["morellon", "pothix"]
   s.email       = ["morellon@gmail.com", "pothix@pothix.com"]
-  s.homepage    = ""
-  s.summary     = %q{Openstack's swift client}
+  s.homepage    = "http://github.com/morellon/openstack-swift"
   s.description = %q{Openstack's swift client}
+  s.summary     = s.description
 
   s.rubyforge_project = "openstack-swift"
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files         = Dir["./**/*"].reject {|file| file =~ /\.git|pkg/}
   s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec", "~> 2.6"
+  s.add_development_dependency "httparty"
+  s.add_development_dependency "ruby-debug19"
 end

data/spec/openstack-swift/client_spec.rb

@@ -0,0 +1,47 @@
+# -*- coding: UTF-8 -*-
+require "spec_helper"
+
+describe "Openstack::Swift::Client" do
+  let!(:swift_dummy_file) do
+    file_path = "/tmp/swift-dummy"
+    File.open(file_path, "w") {|f| f.puts("testfile "*1000)}
+    file_path
+  end
+
+  context "when authenticating" do
+    it "should return authentication error if one of the parameters is incorrect" do
+      expect {
+        Openstack::Swift::Client.new("http://incorrect.com/swift", Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass])
+      }.to raise_error(Openstack::Swift::AuthenticationError)
+    end
+
+    it "shoud not need to authenticate again" do
+      client = Openstack::Swift::Client.new(Openstack::SwiftConfig[:url], Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass])
+      Openstack::Swift::Api.should_not_receive(:auth)
+      client.account_info
+    end
+  end
+
+  context "when authenticated" do
+    subject { Openstack::Swift::Client.new(Openstack::SwiftConfig[:url], Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass]) }
+
+    it "should upload a splitted file and create its manifest" do
+      pending "WTF...Not working for a unknown reason"
+      subject.upload("pothix", swift_dummy_file, {:segments_size => 1024*2})
+      subject.object_info("pothix", "swifty-dummy")["manifest"].should_not be_nil
+    end
+
+    it "should download an splitted file" do
+      content_length = subject.object_info("morellon", "splitted_file")["content_length"].to_i
+      file_path = subject.download("morellon", "splitted_file")
+      File.size(file_path).should == content_length
+    end
+
+    it "should return account's details" do
+      account_info = subject.account_info
+      account_info.should have_key("bytes_used")
+      account_info.should have_key("object_count")
+      account_info.should have_key("container_count")
+    end
+  end
+end

data/spec/openstack-swift/web_api_spec.rb

@@ -0,0 +1,99 @@
+# -*- coding: UTF-8 -*-
+require "spec_helper"
+
+describe Openstack::Swift::Api do
+  context "when authenticating" do
+    it "should authenticate on swift" do
+      expect {
+        subject.auth(Openstack::SwiftConfig[:url], Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass])
+      }.to_not raise_error Openstack::Swift::AuthenticationError
+    end
+
+    it "should raise error for a invalid url" do
+      expect {
+        subject.auth("http://pothix.com/swift", Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass])
+      }.to raise_error Openstack::Swift::AuthenticationError
+    end
+
+    it "should raise error for a invalid pass" do
+      expect {
+        subject.auth(Openstack::SwiftConfig[:url], Openstack::SwiftConfig[:user], "invalidpassword")
+      }.to raise_error Openstack::Swift::AuthenticationError
+    end
+
+    it "should raise error for a invalid user" do
+      expect {
+        subject.auth(Openstack::SwiftConfig[:url], "system:weirduser", Openstack::SwiftConfig[:pass])
+      }.to raise_error Openstack::Swift::AuthenticationError
+    end
+
+    it "should return storage-url, storage-token and auth-token" do
+      subject.auth(Openstack::SwiftConfig[:url], Openstack::SwiftConfig[:user], Openstack::SwiftConfig[:pass]).should have(3).items
+    end
+  end
+
+  context "when authenticated" do
+    let!(:swift_dummy_file){ File.open("/tmp/swift-dummy", "w") {|f| f.puts("test file"*1000)} }
+
+    before do
+      @url, _, @token = subject.auth(
+        Openstack::SwiftConfig[:url],
+        Openstack::SwiftConfig[:user],
+        Openstack::SwiftConfig[:pass]
+      )
+    end
+
+    it "should return account's headers" do
+      account = subject.account(@url, @token)
+      account.should have_key("x-account-bytes-used")
+      account.should have_key("x-account-object-count")
+      account.should have_key("x-account-container-count")
+    end
+
+    it "should return a list of containers" do
+      subject.containers(@url, @token).should be_a(Array)
+    end
+
+    it "should return a list of objects" do
+      subject.objects(@url, @token, "morellon", :delimiter => "/").should be_a(Array)
+    end
+
+    it "should download an object" do
+      subject.download_object(@url, @token, "morellon", "Gemfile").should == "/tmp/swift/morellon/Gemfile"
+    end
+
+    it "should upload an object" do
+      subject.upload_object(@url, @token, "morellon", "/tmp/swift-dummy").code.should == "201"
+    end
+
+    it "should create a new container" do
+      subject.create_container(@url, @token, "pothix_container").should be_true
+    end
+
+    context "when excluding a container" do
+      before { @container = "pothix_container" }
+
+      it "should delete a existent container" do
+        subject.create_container(@url, @token, @container).should be_true
+        subject.delete_container(@url, @token, @container).should be_true
+      end
+
+      it "should raise an error when the container doesn't exist" do
+        expect {
+          subject.delete_container(@url, @token, @container).should be_true
+          subject.delete_container(@url, @token, @container).should be_true
+        }.to raise_error("Could not delete container '#{@container}'")
+      end
+    end
+
+    it "should get the file stat" do
+      subject.upload_object(@url, @token, "morellon", "/tmp/swift-dummy")
+      headers = subject.object_stat(@url, @token, "morellon", "swift-dummy")
+
+      headers["last-modified"].should_not be_blank
+      headers["etag"].should_not be_blank
+      headers["content-type"].should_not be_blank
+      headers["date"].should_not be_blank
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+# -*- coding: UTF-8 -*-
+require "rubygems"
+require "bundler/setup"
+
+require "openstack-swift"
+
+Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each {|file| require file}
+
+# Using a diferent YAML engine on test environment
+YAML::ENGINE::yamler = "syck"
+
+RSpec.configure do |config|
+end

metadata

@@ -1,64 +1,96 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: openstack-swift
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.1.0
   prerelease: 
-  version: 0.0.1
 platform: ruby
-authors: 
+authors:
 - morellon
 - pothix
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-08-10 00:00:00 -03:00
+date: 2011-08-22 00:00:00.000000000 -03:00
 default_executable: 
-dependencies: []
-
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &10022160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: *10022160
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &10021760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *10021760
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &10021280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *10021280
 description: Openstack's swift client
-email: 
+email:
 - morellon@gmail.com
 - pothix@pothix.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- .gitignore
-- Gemfile
-- Rakefile
-- lib/openstack-swift.rb
-- lib/openstack-swift/version.rb
-- openstack-swift.gemspec
+files:
+- ./Gemfile.lock
+- ./config/swift.yml
+- ./Rakefile
+- ./lib/openstack-swift.rb
+- ./lib/openstack-swift/swift_config.rb
+- ./lib/openstack-swift/errors.rb
+- ./lib/openstack-swift/api.rb
+- ./lib/openstack-swift/client.rb
+- ./lib/openstack-swift/version.rb
+- ./Gemfile
+- ./openstack-swift.gemspec
+- ./spec/openstack-swift/client_spec.rb
+- ./spec/openstack-swift/web_api_spec.rb
+- ./spec/spec_helper.rb
 has_rdoc: true
-homepage: ""
+homepage: http://github.com/morellon/openstack-swift
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: openstack-swift
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Openstack's swift client
 test_files: []
-

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-.bundle
-Gemfile.lock
-pkg/*