att-swift 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a7af7bfed0c75cb7448a6d6551aa6106af7a694
+  data.tar.gz: a7e6de50f392136adec0125a0300dfce947efe23
+SHA512:
+  metadata.gz: aac182861476e6c7f88095b0dfff587efd24e5f48b0323d7273ac2336344630434a9672a48dd63102fd11ed537995e26d2c67076a72d9c80249ac7db7cf61047
+  data.tar.gz: 43d852f419dc752077a6252d31358397f0d2f2664b5e96282c540dadd70cc1709cb03977cfb7fa4d482efb12ab07c48e1e7d1c587af5c1cfcb5ca1219986bbf0

checksums.yaml.gz.sig

@@ -0,0 +1,6 @@
+{}÷!�6�}���j�0�/�9���F�#C��MQ#��U�Pt�1)X����RG���Pޮ���a��J=�ZV�.�O�������1p1��+p)#��mH�\acb�V�r5�Zl�`���3n!�l�8��
+�	 �h�!��
+-��
+^��Y�V@d�.��
+��AJ��p؝�Q|`o�ue7$�18�)��:�D~��kU�kxd|�f��][
+��p[�_{��/�*�ݝ?Q�

data/.autotest

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+  at.testlib = 'minitest/autorun'
+  at.add_exception '.git'
+
+  def at.path_to_classname s
+    sep = File::SEPARATOR
+    f = s.sub(/^test#{sep}/, '').sub(/\.rb$/, '').split(sep)
+    f = f.map { |path| path.split(/_|(\d+)/).map { |seg| seg.capitalize }.join }
+    f = f.map { |path| path =~ /^Test/ ? path : "Test#{path}"  }
+    f.join('::').gsub('Att', 'ATT')
+  end
+end
+

data/History.rdoc

@@ -0,0 +1,5 @@
+=== 1.0 / 2012-08-21
+
+* Major enhancements
+  * Birthday!
+

data/LICENSE.rdoc

@@ -0,0 +1,20 @@
+(The MIT License)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,9 @@
+.autotest
+History.rdoc
+LICENSE.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+bin/swift_dump
+lib/att/swift.rb
+test/test_att_swift.rb

data/README.rdoc

@@ -0,0 +1,33 @@
+= att-swift
+
+home :: https://github.com/att-cloud/att-swift
+bugs :: https://github.com/att-cloud/att-swift/issues
+docs :: http://docs.seattlerb.org/att-swift
+
+== Description
+
+A wrapper for OpenStack Object Storage v1 (aka Swift).  Swift provides
+redundant storage similar to AWS S3.  This gem is a wrapper around the Object
+Storage RESTful API and provides the ability to manage containers
+(directories) and objects (files).
+
+== Features and Problems
+
+* Supports basic container actions
+* Supports basic object actions
+* Missing easy large object creation
+* Missing most metadata actions
+
+== Install
+
+  sudo gem install att-swift
+
+== Developers
+
+After checking out the source, run:
+
+  $ rake newb
+
+This task will install any missing dependencies, run the tests/specs,
+and generate the RDoc.
+

data/Rakefile

@@ -0,0 +1,21 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.plugin :git
+Hoe.plugin :minitest
+Hoe.plugin :travis
+
+Hoe.spec 'att-swift' do
+  developer 'Eric Hodel', 'drbrain@segment7.net'
+
+  self.licenses << 'MIT'
+
+  rdoc_locations <<
+    'docs.seattlerb.org:/data/www/docs.seattlerb.org/att-swift/'
+
+  extra_deps << ['net-http-persistent', '~> 2.7']
+end
+
+# vim: syntax=ruby

data/bin/swift_dump

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'att/swift'
+
+uri  = ARGV.shift
+user = ARGV.shift
+key  = ARGV.shift
+path = ARGV.shift
+
+abort "swift_dump AUTH_URI USER KEY container[/object]" unless path
+
+swift = ATT::Swift.new uri, user, key
+
+container, object = path.split '/', 2
+
+if object then
+  swift.read_object container, object do |res|
+    res.read_body do |chunk|
+      $stdout.write chunk
+    end
+  end
+else
+  swift.paginate_objects container do |object_info|
+    puts object_info['name']
+  end
+end
+

data/lib/att/swift.rb

@@ -0,0 +1,582 @@
+require 'cgi'
+require 'digest'
+require 'json'
+require 'net/http/persistent'
+
+module ATT # :nodoc:
+end
+
+##
+# A wrapper for OpenStack Object Storage v1 (aka Swift).  Swift provides
+# redundant storage similar to AWS S3.  This gem is a wrapper around the
+# Object # Storage RESTful API and provides the ability to manage containers
+# (directories) and objects (files).
+#
+# Example:
+#
+#   require 'pp'
+#   require 'att/swift'
+#
+#   auth_uri = 'http://swift.example/auth/'
+#   swift = ATT::Swift.new auth_uri, 'username', 'password'
+#
+#   pp swift.containers
+#
+# See also http://docs.openstack.org/api/openstack-object-storage/1.0/content/
+
+class ATT::Swift
+
+  ##
+  # The version of att-swift you are using
+
+  VERSION = '1.0'
+
+  attr_accessor :http # :nodoc:
+
+  attr_reader :auth_token  # :nodoc:
+  attr_reader :storage_uri # :nodoc:
+
+  ##
+  # Base error class for exceptions raised by Swift
+
+  class Error < RuntimeError
+
+    ##
+    # Net::HTTP response for this error
+
+    attr_reader :http_response
+
+    ##
+    # Creates a new Swift::Error for a Net::HTTP +response+ with the
+    # additional +message+ describing why the response caused the error.
+
+    def initialize response, message
+      @http_response = response
+      super "#{message} - #{response.code}"
+    end
+
+  end
+
+  ##
+  # Creates a new Swift object for communication with the server at
+  # +auth_uri+.  +user+ and +key+ are your credentials.
+  #
+  # For AT&T Cloud storage:
+  #
+  #   swift = ATT::Swift.new \
+  #     'https://data.iad1.attstorage.com/auth/',
+  #     'tennant:username', 'password'
+
+  def initialize auth_uri, user, key
+    auth_uri = URI auth_uri unless URI === auth_uri
+
+    @auth_uri = auth_uri
+    @user     = user
+    @key      = key
+
+    @http = Net::HTTP::Persistent.new 'att-swift'
+
+    @storage_uri = nil
+    @auth_token  = nil
+  end
+
+  ##
+  # Authenticates you with the swift server.  This method is called
+  # automatically by other API methods.
+
+  def authenticate
+    return if @auth_token
+
+    res = get @auth_uri + 'v1.0', 'X-Auth-User' => @user, 'X-Auth-Key' => @key
+
+    case res
+    when Net::HTTPSuccess
+      @auth_token  = res['X-Auth-Token']
+
+      storage_uri = res['X-Storage-Url']
+      storage_uri << '/' unless storage_uri.end_with? '/'
+      @storage_uri = URI storage_uri
+
+      @http.override_headers['X-Auth-Token'] = @auth_token
+    else
+      raise Error.new(res, 'authentication failed')
+    end
+  end
+
+  ##
+  # Like paginate_objects, but yields chunks of up to +limit+ object
+  # information at a time from the swift server to the given block.
+  # +marker+ may be used to start pagination after a particular entry.
+  #
+  #   swift.chunk_objects 'container' do |object_infos|
+  #     object_infos.each do |object_info|
+  #       # ...
+  #     end
+  #   end
+
+  def chunk_objects container, marker = nil, limit = 1_000
+    return enum_for __method__, container, marker, limit unless block_given?
+
+    loop do
+      chunk = objects container, marker, limit
+
+      break if chunk.empty?
+
+      yield chunk
+
+      marker = chunk.last['name']
+    end
+
+    self
+  end
+
+  ##
+  # Retrieves the containers for your server.  Example output:
+  #
+  #  [{"name" => "public_bucket", "count" => 2, "bytes" => 9},
+  #   {"name" => "public_bucket_segments", "count" => 22, "bytes" => 21875}]
+
+  def containers marker = nil, limit = nil
+    authenticate
+
+    params = { 'format' => 'json' }
+    params['marker'] = marker if marker
+    params['limit']  = limit  if limit
+
+    uri = @storage_uri + "?#{escape_params params}"
+
+    res = get uri
+
+    case res
+    when Net::HTTPSuccess then
+      JSON.parse res.body
+    else
+      raise Error.new(res, 'error listing containers')
+    end
+  end
+
+  ##
+  # Copies +source_object+ in +source_container+ to +destination_object+ in
+  # +destination_container+.
+  #
+  # Returns true if the copy was successful, false if the source object was
+  # not found or the destination container did not exist.
+
+  def copy_object source_container,      source_object,
+                  destination_container, destination_object
+    authenticate
+
+    source_path      = "#{source_container}/#{source_object}"
+    destination_path = "#{destination_container}/#{destination_object}"
+
+    uri = @storage_uri + destination_path
+
+    req = Net::HTTP::Put.new uri.request_uri
+    req['X-Copy-From'] = "/#{source_path}"
+    req.body = ''
+
+    res = @http.request uri, req
+
+    case res
+    when Net::HTTPSuccess then
+      true
+    when Net::HTTPNotFound
+      false
+    else
+      raise
+        Error.new(res, "error copying #{source_path} to #{destination_path}")
+    end
+  end
+
+  ##
+  # HTTP DELETE
+
+  def delete uri, headers = {} # :nodoc:
+    request Net::HTTP::Delete, uri, headers
+  end
+
+  ##
+  # Creates a new +container+.  Returns true if the container was created,
+  # false if it already existed.
+
+  def create_container container
+    authenticate
+
+    uri = @storage_uri + container
+
+    res = put uri
+
+    case res
+    when Net::HTTPCreated then
+      true
+    when Net::HTTPSuccess then
+      false
+    else
+      raise Error.new(res, 'error creating container')
+    end
+  end
+
+  ##
+  # Deletes +container+.  Returns true if the container existed, false if not.
+  # Raises an exception otherwise.
+
+  def delete_container container
+    authenticate
+
+    uri = @storage_uri + container
+
+    res = delete uri
+
+    case res
+    when Net::HTTPNoContent then
+      true
+    when Net::HTTPNotFound then
+      false
+    when Net::HTTPConflict then
+      raise Error.new(res, "container #{container} is not empty")
+    else
+      raise Error.new(res, "error deleting container")
+    end
+  end
+
+  ##
+  # Deletes +object+ from +container+  Returns true if the object existed, false
+  # if not.  Raises an exception otherwise.
+
+  def delete_object container, object
+    authenticate
+
+    uri = @storage_uri + "#{container}/#{object}"
+
+    res = delete uri
+
+    case res
+    when Net::HTTPNoContent then
+      true
+    when Net::HTTPNotFound then
+      false
+    else
+      raise Error.new(res, "error deleting #{object} in #{container}")
+    end
+  end
+
+  def escape_params params # :nodoc:
+    params.map do |k, v|
+      [CGI.escape(k.to_s), CGI.escape(v.to_s)].join '='
+    end.compact.join '&'
+  end
+
+  ##
+  # HTTP GET
+
+  def get uri, headers = {}, &block # :nodoc:
+    request Net::HTTP::Get, uri, headers, &block
+  end
+
+  ##
+  # HTTP HEAD
+
+  def head uri, headers = {} # :nodoc:
+    request Net::HTTP::Head, uri, headers
+  end
+
+  def inspect # :nodoc:
+    "#<%s:0x%x %s %s>" % [
+      self.class,
+      object_id,
+      @auth_uri,
+      @user
+    ]
+  end
+
+  ##
+  # Lists objects in +container+.  +marker+ lists objects after the given
+  # name, +limit+ restricts the object listing to that many items.
+  #
+  # Example output:
+  #
+  #   [{"name"=>"test-0",
+  #     "hash"=>"b1946ac92492d2347c6235b4d2611184",
+  #     "bytes"=>6,
+  #     "content_type"=>"application/octet-stream",
+  #     "last_modified"=>"2012-08-22T19:24:03.102100"},
+  #    {"name"=>"test-1",
+  #     "hash"=>"802eeebb01b647913806a870cbb5394a",
+  #     "bytes"=>52707,
+  #     "content_type"=>"application/octet-stream",
+  #     "last_modified"=>"2012-08-22T19:32:24.474980"},
+  #    {"name"=>"test-2",
+  #     "hash"=>"90affbd9a1954ec9ff029b7ad7183a16",
+  #     "bytes"=>5,
+  #     "content_type"=>"application/octet-stream",
+  #     "last_modified"=>"2012-08-22T21:10:05.258080"}]
+
+  def objects container, marker = nil, limit = nil
+    authenticate
+
+    params = { 'format' => 'json' }
+    params['marker'] = marker if marker
+    params['limit']  = limit  if limit
+
+    uri = @storage_uri + "#{container}?#{escape_params params}"
+
+    res = get uri
+
+    case res
+    when Net::HTTPSuccess then
+      JSON.parse res.body
+    else
+      raise Error.new(res, "error retrieving object list in #{container}")
+    end
+  end
+
+  ##
+  # Retrieves information for +object+ in +container+ including metadata.
+  # Example result:
+  #
+  #   {
+  #     'metadata' => {
+  #       'meat' => 'Bacon',
+  #     },
+  #
+  #     'content-length' => '3072',
+  #     'content-type'   => 'application/octet-stream',
+  #     'etag'           => 'a2a5648d09a83c6a85ddd62e4a22309c',
+  #     'last-modified'  => 'Wed, 22 Aug 2012 22:51:28 GMT',
+  #   }
+
+  def object_info container, object
+    authenticate
+
+    uri = @storage_uri + "#{container}/#{object}"
+
+    res = head uri
+
+    case res
+    when Net::HTTPSuccess then
+      metadata = {}
+      info     = { 'metadata' => metadata }
+
+      res.each do |name, value|
+        case name
+        when /^accept/i, 'connection', 'date' then
+          next
+        when /^x-object-meta-(.*)/ then
+          metadata[$1] = value
+        when /^x-/ then
+          next
+        else
+          info[name] = value
+        end
+      end
+
+      info
+    when Net::HTTPNotFound then
+      nil
+    else
+      raise Error.new(res,
+                      "error retrieving metadata for #{object} in #{container}")
+    end
+  end
+
+  ##
+  # Retrieves metadata for +object+ in +container+.  See also
+  # object_info.  Example result:
+  #
+  #   'metadata' => {
+  #     'meat' => 'Bacon',
+  #   }
+
+  def object_metadata container, object
+    info = object_info container, object
+
+    return unless info
+
+    info['metadata']
+  end
+
+  ##
+  # Like #objects, but only retrieves +limit+ objects at a time from the
+  # swift server and yields the object information to the given block.
+  # +marker+ may be used to start pagination after a particular entry.
+  #
+  #   swift.paginate_objects 'container' do |object_info|
+  #     p object_info.name
+  #   end
+
+  def paginate_objects container, marker = nil, limit = 1_000
+    return enum_for __method__, container, marker, limit unless block_given?
+
+    chunk_objects container, marker, limit do |chunk|
+      chunk.each do |object_info|
+        yield object_info
+      end
+    end
+
+    self
+  end
+
+  ##
+  # HTTP POST
+
+  def post uri, headers = {} # :nodoc:
+    request Net::HTTP::Post, uri, headers
+  end
+
+  ##
+  # HTTP PUT
+
+  def put uri, headers = {} # :nodoc:
+    request Net::HTTP::Put, uri, headers
+  end
+
+  ##
+  # Reads +object+ from +container+.  Unless the response is HTTP 200 OK an
+  # exception is raised.
+  #
+  # If no block is given the response body is read, checked for a matching MD5
+  # checksum and returned.
+  #
+  # If a block is given the response is yielded for custom handling and no MD5
+  # checksum is calculated.
+  #
+  # Example:
+  #
+  #   swift.read_object 'test-container', 'test-object' do |res|
+  #     open destination, 'wb' do |io|
+  #       res.read_body do |chunk|
+  #         io.write chunk
+  #       end
+  #     end
+  #   end
+
+  def read_object container, object
+    authenticate
+
+    uri = @storage_uri + "#{container}/#{object}"
+
+    get uri do |res|
+      case res
+      when Net::HTTPOK then
+        if block_given? then
+          yield res
+        else
+          body = res.body
+
+          if res['ETag'] != Digest::MD5.hexdigest(body) then
+            raise Error.new(res,
+                            "checksum retrieving object #{object} in #{container}")
+          end
+
+          body
+        end
+      when Net::HTTPNotFound then
+        raise Error.new(res, "object #{object} in #{container} not found")
+      else
+        raise Error.new(res, "error retrieving object #{object} in #{container}")
+      end
+    end
+  end
+
+  ##
+  # net-http-persistent request wrapper
+
+  def request klass, uri, headers, &block # :nodoc:
+    path = klass::REQUEST_HAS_BODY ? uri.path : uri.request_uri
+
+    req = klass.new path, headers
+
+    @http.request uri, req, &block
+  end
+
+  ##
+  # Sets the metadata for +object+ in +container+.  Existing metadata will be
+  # overwritten.
+
+  def set_object_metadata container, object, metadata = {}
+    authenticate
+
+    uri = @storage_uri + "#{container}/#{object}"
+
+    headers = {}
+
+    metadata.each do |name, value|
+      headers["X-Object-Meta-#{name}"] = value
+    end
+
+    res = post uri, headers
+
+    case res
+    when Net::HTTPSuccess then
+      true
+    else
+      raise Error.new(res,
+                      "error setting metadata on #{object} in #{container}")
+    end
+  end
+
+  ##
+  # Writes to +object+ in +container+.  +content+ may be a String or IO-like
+  # object.  If +content+ is a String the response ETag is checked against the
+  # MD5 hash of the local copy.
+  #
+  # A block may be given instead of +content+.  You will be given a pipe you
+  # can write the content to:
+  #
+  #   r = Random.new
+  #
+  #   swift.write_object 'test-container', 'test-object' do |io|
+  #     r.rand(10).times do
+  #       io.write r.bytes r.rand 10_000
+  #     end
+  #   end
+  #
+  # In all cases the response ETag is returned when the content was
+  # successfully uploaded.
+
+  def write_object container, object, content = nil
+    raise ArgumentError, 'provide block or content' if
+      content and block_given?
+
+    if block_given? then
+      content, write = IO.pipe
+
+      Thread.start do
+        begin
+          yield write
+        ensure
+          write.close
+        end
+      end
+    end
+
+    authenticate
+
+    uri = @storage_uri + "#{container}/#{object}"
+
+    req = Net::HTTP::Put.new uri.path
+
+    case content
+    when String then
+      req.body = content
+      req['ETag'] = Digest::MD5.hexdigest content
+    else
+      req['Transfer-Encoding'] = 'chunked'
+      req.body_stream = content
+    end
+
+    req.content_type = 'application/octet-stream'
+
+    res = @http.request uri, req
+
+    case res
+    when Net::HTTPCreated then
+      res['ETag']
+    else
+      raise Error.new(res,
+                      "error creating object #{object} in container #{container}")
+    end
+  end
+
+end
+