qoobaa-stree 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,6 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+*.gem

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Jakub Kuźma, Mirosław Boruta
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,48 @@
+= Stree
+
+* repository: http://github.com/qoobaa/stree
+* issue tracker: http://github.com/qoobaa/stree/issues
+* rdoc: http://qoobaa.github.com/stree
+
+== Usage
+
+Coming soon, see Rdoc documentation.
+
+== Command tool usage
+
+You have to pass access key id (-a) and secret access key (-s) to the
+command line tool. Stree reads ACCESS_KEY_ID and SECRET_ACCESS_KEY
+environment variables and uses them by default, so if you don't want
+to pass them each time, export them (e.g. in .bashrc file).
+
+* list buckets
+  stree bucket
+
+* create bucket
+  stree bucket add name-of-bucket
+
+* list objects in bucket
+  stree bucket show name-of-bucket
+
+* destroy bucket
+  stree bucket remove name-of-bucket
+
+* list objects (in all buckets)
+  stree object
+
+* show the object information
+  stree object show bucket_name/path/to/object.extension
+
+* download the content of the object
+  stree object show bucket_name/path/to/object.extension filename_to_store_the_content.extension
+
+* show the content of the object to STDOUT
+  stree object show bucket_name/path/to/object.extension -
+
+You can also pass --help to the commmands like:
+  stree bucket add --help
+  stree object show --help
+
+== Copyright
+
+Copyright (c) 2009 Jakub Kuźma, Mirosław Boruta. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "stree"
+    gem.summary = %Q{Library for accessing S3 objects and buckets, with command line tool}
+    gem.email = "qoobaa@gmail.com"
+    gem.homepage = "http://github.com/qoobaa/stree"
+    gem.authors = ["Jakub Kuźma", "Mirosław Boruta"]
+    gem.add_dependency "trollop", ">=1.14"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "stree #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/stree

@@ -0,0 +1,187 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path(File.dirname(__FILE__) + "/../lib")
+
+require "trollop"
+require "stree"
+
+# HELPER METHODS
+
+include Stree
+
+def list_buckets(service)
+  service.buckets.each do |bucket|
+    puts bucket.name
+  end
+end
+
+def create_bucket(service, name, location)
+  service.buckets.build(name).save(location)
+end
+
+def destroy_bucket(service, name)
+  service.buckets.find(name).destroy
+end
+
+def show_bucket(service, name, options = {})
+  service.buckets.find(name).objects.find_all.each do |object|
+    puts "#{name}/#{object.key}"
+  end
+end
+
+def list_objects(service)
+  service.buckets.each do |bucket|
+    bucket.objects.each do |object|
+      puts "#{bucket.name}/#{object.key}"
+    end
+  end
+end
+
+def create_object(service, name, file_name, options = {})
+  bucket_name, object_name = name.split("/", 2)
+  object = service.buckets.find(bucket_name).objects.build(object_name)
+  object.content_type = options[:type]
+  object.content_encoding = options[:encoding]
+  object.content_disposition = options[:disposition]
+  object.acl = options[:acl]
+  object.content = File.new(file_name)
+  object.save
+end
+
+def destroy_object(service, name)
+  bucket_name, object_name = name.split("/", 2)
+  object = service.buckets.find(bucket_name).objects.find(object_name)
+  object.destroy
+end
+
+def show_object(service, name, file_name = nil)
+  bucket_name, object_name = name.split("/", 2)
+  object = service.buckets.find(bucket_name).objects.find_first(object_name)
+  puts "         object:   #{object.name}/#{object.key}"
+  puts "   content type:   #{object.content_type}"
+  puts "           size:   #{object.size}"
+  puts "           etag:   #{object.etag}"
+  puts "  last modified:   #{object.last_modified}"
+  if file_name
+    if file_name == "-"
+      puts object.content
+    else
+      File.open(file_name, "wb") do |file|
+        file.write(object.content)
+      end
+    end
+  end
+end
+
+# COMMAND LINE PARSER
+
+ACCESS_KEY_ID = ENV["ACCESS_KEY_ID"]
+SECRET_ACCESS_KEY = ENV["SECRET_ACCESS_KEY"]
+COMMANDS = %w(bucket object)
+BUCKET_SUBCOMMANDS = %w(add remove show)
+OBJECT_SUBCOMMANDS = %w(add remove show)
+
+global_options = Trollop::options do
+  banner "stree - s3 command line tool"
+  opt :access_key_id, "Your access key id to AWS", :type => :string, :default => ACCESS_KEY_ID
+  opt :secret_access_key, "Your secret access key to AWS", :type => :string, :default => SECRET_ACCESS_KEY
+  opt :debug, "Debug mode", :type => :flag, :default => false
+  stop_on COMMANDS
+end
+
+Trollop::die "No access key id given" unless global_options[:access_key_id]
+Trollop::die "No secret access key given" unless global_options[:secret_access_key]
+
+service = Service.new(:access_key_id => global_options[:access_key_id],
+                      :secret_access_key => global_options[:secret_access_key],
+                      :debug => global_options[:debug])
+
+command = ARGV.shift
+
+begin
+  case command
+  when "bucket"
+    command_options = Trollop::options do
+      banner "manage buckets"
+      stop_on BUCKET_SUBCOMMANDS
+    end
+    subcommand = ARGV.shift
+    case subcommand
+    when "add"
+      subcommand_options = Trollop::options do
+        banner "add bucket"
+        opt :location, "Location of the bucket - EU or US", :default => "US", :type => :string
+      end
+      name = ARGV.shift
+      Trollop::die "Bucket has not been added because of unknown error" unless create_bucket(service, name, subcommand_options[:location])
+    when "remove"
+      subcommand_options = Trollop::options do
+        banner "remove bucket"
+      end
+      name = ARGV.shift
+      Trollop::die "Bucket name must be given" if name.nil? or name.empty?
+      Trollop::die "Bucket has not been removed because of unknown error" unless destroy_bucket(service, name)
+    when "show"
+      subcommand_options = Trollop::options do
+        banner "show bucket"
+        opt :prefix, "Limits the response to keys which begin with the indicated prefix", :type => :string
+        opt :marker, "Indicates where in the bucket to begin listing", :type => :string
+        opt :max_keys, "The maximum number of keys you'd like to see", :type => :integer
+        opt :delimiter, "Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element", :type => :string
+      end
+      name = ARGV.shift
+      Trollop::die "Bucket name must be given" if name.nil? or name.empty?
+      show_bucket(service, name, subcommand_options)
+    when nil
+      list_buckets(service)
+    else
+      Trollop::die "Unknown subcommand: #{subcommand.inspect}"
+    end
+  when "object"
+    command_options = Trollop::options do
+      banner "manage objects"
+      stop_on OBJECT_SUBCOMMANDS
+    end
+    subcommand = ARGV.shift
+    case subcommand
+    when "add"
+      subcommand_options = Trollop::options do
+        banner "object add s3_object_name local_file_name"
+        opt :type, "A standard MIME type describing the format of the contents", :default => "binary/octet-stream"
+        opt :disposition, "Specifies presentational information for the object", :type => :string
+        opt :encoding, "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied in order to obtain the media-type referenced by the Content-Type header field", :type => :string
+        opt :acl, "The canned ACL to apply to the object. Options include private, public-read, public-read-write, and authenticated-read", :type => :string
+      end
+      name = ARGV.shift
+      Trollop::die "No object name given" if name.nil? or name.empty?
+      file_name = ARGV.shift
+      Trollop::die "No file name given" if file_name.nil? or file_name.empty?
+      Trollop::die "Object has not been added because of unknown error" unless create_object(service, name, file_name, subcommand_options)
+    when "remove"
+      subcommand_options = Trollop::options do
+        banner "object remove s3_object_name"
+      end
+      name = ARGV.shift
+      Trollop::die "No object name given" if name.nil? or name.empty?
+      Trollop::die "Object has not been removed because of unknown error" unless destroy_object(service, name)
+    when "show"
+      subcommand_options = Trollop::options do
+        banner "object show s3_object_name optional_file_name"
+      end
+      name = ARGV.shift
+      Trollop::die "No object name given" if name.nil? or name.empty?
+      file_name = ARGV.shift
+      show_object(service, name, file_name)
+    when nil
+      list_objects(service)
+    else
+      Trollop::die "Unknown subcommand: #{subcommand.inspect}"
+    end
+  when nil
+    Trollop::die "No command given"
+  else
+    Trollop::die "Unknown command #{command.inspect}"
+  end
+rescue Error::ResponseError => e
+  Trollop::die e.message.sub(/\.+\Z/, "")
+end

data/extra/stree_backend.rb

@@ -0,0 +1,159 @@
+require "singleton"
+require "stree"
+
+# Stree Backend for attachment-fu plugin. After installing attachment-fu
+# plugin, copy the file to:
+# +vendor/plugins/attachment-fu/lib/technoweenie/attachment_fu/backends+
+#
+# To configure StreeBackend create initializer file in your Rails
+# application, e.g. +config/initializers/stree_backend.rb+.
+#
+#   Technoweenie::AttachmentFu::Backends::StreeBackend.configuration do |config|
+#     config.access_key_id = "..." # your access key id
+#     config.secret_access_key = "..." # your secret access key
+#     config.bucket_name = "..." # default bucket name to store attachments
+#     config.use_ssl = false # pass true if you want to communicate via SSL
+#   end
+
+module Technoweenie
+  module AttachmentFu
+    module Backends
+      module StreeBackend
+
+        # StreeBackend configuration class
+        class Configuration
+          include Singleton
+
+          ATTRIBUTES = [:access_key_id, :secret_access_key, :use_ssl, :bucket_name]
+
+          attr_accessor *ATTRIBUTES
+        end
+
+        # Method used to configure StreeBackend, see the example above
+        def self.configuration
+          if block_given?
+            yield Configuration.instance
+          end
+          Configuration.instance
+        end
+
+        # :nodoc:
+        def self.included(base)
+          include Stree
+
+          service = Service.new(:access_key_id => configuration.access_key_id,
+                                :secret_access_key => configuration.secret_access_key,
+                                :use_ssl => configuration.use_ssl)
+
+          bucket_name = base.attachment_options[:bucket_name] || configuration.bucket_name
+
+          base.cattr_accessor :bucket
+          base.bucket = service.buckets.build(bucket_name) # don't connect
+
+          base.before_update :rename_file
+        end
+
+        # The attachment ID used in the full path of a file
+        def attachment_path_id
+          ((respond_to?(:parent_id) && parent_id) || id).to_s
+        end
+
+        # The pseudo hierarchy containing the file relative to the bucket name
+        # Example: <tt>:table_name/:id</tt>
+        def base_path
+          [attachment_options[:path_prefix], attachment_path_id].join("/")
+        end
+
+        # The full path to the file relative to the bucket name
+        # Example: <tt>:table_name/:id/:filename</tt>
+        def full_filename(thumbnail = nil)
+          [base_path, thumbnail_name_for(thumbnail)].join("/")
+        end
+
+        # All public objects are accessible via a GET request to the S3 servers. You can generate a
+        # url for an object using the s3_url method.
+        #
+        #   @photo.s3_url
+        #
+        # The resulting url is in the form: <tt>http(s)://:server/:bucket_name/:table_name/:id/:file</tt> where
+        # the <tt>:server</tt> variable defaults to <tt>AWS::S3 URL::DEFAULT_HOST</tt> (s3.amazonaws.com) and can be
+        # set using the configuration parameters in <tt>RAILS_ROOT/config/amazon_s3.yml</tt>.
+        #
+        # The optional thumbnail argument will output the thumbnail's filename (if any).
+        def s3_url(thumbnail = nil)
+          if attachment_options[:cname]
+            ["#{s3_protocol}#{bucket.name}", full_filename(thumbnail)].join("/")
+          else
+            ["#{s3_protocol}#{s3_hostname}#{bucket.path_prefix}", full_filename(thumbnail)].join("/")
+          end
+        end
+        alias :public_url :s3_url
+        alias :public_filename :s3_url
+
+        # Name of the bucket used to store attachments
+        def bucket_name
+          self.class.bucket.name
+        end
+
+        # :nodoc:
+        def create_temp_file
+          write_to_temp_file current_data
+        end
+
+        # :nodoc:
+        def current_data
+          # Object.value full_filename, bucket_name
+          object = self.class.bucket.find(full_filename)
+          object.content
+        end
+
+        # Returns http:// or https:// depending on use_ssl setting
+        def s3_protocol
+          attachment_options[:use_ssl] ?  "https://" : "http://"
+        end
+
+        # Returns hostname of the bucket
+        # e.g. +bucketname.com.s3.amazonaws.com+. Additionally you can
+        # pass :cname => true option in has_attachment method to
+        # return CNAME only, e.g. +bucketname.com+
+        def s3_hostname
+          attachment_options[:cname] ? self.class.bucket.name : self.class.bucket.host
+        end
+
+        protected
+
+        # Frees the space in S3 bucket, used by after_destroy callback
+        def destroy_file
+          object = self.class.bucket.objects.find(full_filename)
+          object.destroy
+        end
+
+        # Renames file if filename has been changed - copy the file to
+        # new key and delete old one
+        def rename_file
+          return unless filename_changed?
+
+          old_full_filename = [base_path, filename_was].join("/")
+
+          object = self.class.bucket.objects.find(old_full_filename)
+          new_object = object.copy(:key => full_filename, :acl => attachment_options[:acl])
+          object.destroy
+          true
+        end
+
+        # Saves the file to storage
+        def save_to_storage
+          if save_attachment?
+            object = self.class.bucket.objects.build(full_filename)
+
+            object.content_type = content_type
+            object.acl = attachment_options[:acl]
+            object.content = temp_path ? File.open(temp_path) : temp_data
+            object.save
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/stree/bucket.rb

@@ -0,0 +1,192 @@
+module Stree
+  class Bucket
+    extend Roxy::Moxie
+    extend Forwardable
+
+    attr_reader :name, :service
+
+    def_instance_delegators :service, :service_request
+
+    # Retrieves the bucket information from the server. Raises an
+    # Stree::Error exception if the bucket doesn't exist or you don't
+    # have access to it, etc.
+    def retrieve
+      bucket_request(:get, :params => { :max_keys => 0 })
+      self
+    end
+
+    # Returns location of the bucket, e.g. "EU"
+    def location(reload = false)
+      if reload or @location.nil?
+        response = bucket_request(:get, :params => { :location => nil })
+        @location = parse_location(response.body)
+      else
+        @location
+      end
+    end
+
+    # Compares the bucket with other bucket. Returns true if the key
+    # of the objects are the same, and both have the same buckets (see
+    # bucket equality)
+    def ==(other)
+      self.name == other.name and self.service == other.service
+    end
+
+    # Similar to retrieve, but catches NoSuchBucket exceptions and
+    # returns false instead.
+    def exists?
+      retrieve
+      true
+    rescue Error::NoSuchBucket
+      false
+    end
+
+    # Destroys given bucket. Raises an BucketNotEmpty exception if the
+    # bucket is not empty. You can destroy non-empty bucket passing
+    # true (to force destroy)
+    def destroy(force = false)
+      bucket_request(:delete)
+      true
+    rescue Error::BucketNotEmpty
+      if force
+        objects.destroy_all
+        retry
+      else
+        raise
+      end
+    end
+
+    # Saves the newly built bucket. Optionally you can pass location
+    # of the bucket (:eu or :us)
+    def save(location = nil)
+      location = location.to_s.upcase if location
+      options = { :headers => {} }
+      if location and location != "US"
+        options[:body] = "<CreateBucketConfiguration><LocationConstraint>#{location}</LocationConstraint></CreateBucketConfiguration>"
+        options[:headers][:content_type] = "application/xml"
+      end
+      bucket_request(:put, options)
+      true
+    end
+
+    # Returns true if the name of the bucket can be used like VHOST
+    # name. If the bucket contains characters like underscore it can't
+    # be used as VHOST (e.g. bucket_name.s3.amazonaws.com)
+    def vhost?
+      "#@name.#{HOST}" =~ /\A#{URI::REGEXP::PATTERN::HOSTNAME}\Z/
+    end
+
+    # Returns host name of the bucket according (see vhost?)
+    def host
+      vhost? ? "#@name.#{HOST}" : "#{HOST}"
+    end
+
+    # Returns path prefix for non VHOST bucket. Path prefix is used
+    # instead of VHOST name,
+    # e.g. "bucket_name/"
+    def path_prefix
+      vhost? ? "" : "#@name/"
+    end
+
+    # Returns the objects in the bucket and caches the result (see
+    # reload).
+    def objects(reload = false)
+      if reload or @objects.nil?
+        @objects = fetch_objects
+      else
+        @objects
+      end
+    end
+
+    proxy :objects do
+
+      # Builds the object in the bucket with given key
+      def build(key)
+        Object.new(proxy_owner, key)
+      end
+
+      # Finds first object with given name or raises the exception if
+      # not found
+      def find_first(name)
+        object = build(name)
+        object.retrieve
+      end
+      alias :find :find_first
+
+      # Finds the objects in the bucket.
+      # ==== Options:
+      # +prefix+:: Limits the response to keys which begin with the indicated prefix
+      # +marker+:: Indicates where in the bucket to begin listing
+      # +max_keys+:: The maximum number of keys you'd like to see
+      # +delimiter+:: Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element
+      def find_all(options = {})
+        proxy_owner.send(:fetch_objects, options)
+      end
+
+      # Reloads the object list (clears the cache)
+      def reload
+        proxy_owner.objects(true)
+      end
+
+      # Destroys all keys in the bucket
+      def destroy_all
+        proxy_target.each do |object|
+          object.destroy
+        end
+      end
+    end
+
+    def inspect #:nodoc:
+      "#<#{self.class}:#{name}>"
+    end
+
+    def initialize(service, name) #:nodoc:
+      self.service = service
+      self.name = name
+    end
+
+    private
+
+    attr_writer :service
+
+    def name=(name)
+      raise ArgumentError.new("Invalid bucket name: #{name}") unless name_valid?(name)
+      @name = name
+    end
+
+    def bucket_request(method, options = {})
+      path = "#{path_prefix}#{options[:path]}"
+      service_request(method, options.merge(:host => host, :path => path))
+    end
+
+    def fetch_objects(options = {})
+      response = bucket_request(:get, options)
+      parse_objects(response.body)
+    end
+
+    def parse_objects(xml_body)
+      xml = XmlSimple.xml_in(xml_body)
+      objects_attributes = xml["Contents"]
+      if objects_attributes
+        objects_attributes.map do |object_attributes|
+          Object.new(self,
+                     object_attributes["Key"].first,
+                     :etag => object_attributes["ETag"].first,
+                     :last_modified => object_attributes["LastModified"].first,
+                     :size => object_attributes["Size"].first)
+        end
+      else
+        []
+      end
+    end
+
+    def parse_location(xml_body)
+      xml = XmlSimple.xml_in(xml_body)
+      xml["content"]
+    end
+
+    def name_valid?(name)
+      name =~ /\A[a-z0-9][a-z0-9\._-]{2,254}\Z/ and name !~ /\A#{URI::REGEXP::PATTERN::IPV4ADDR}\Z/
+    end
+  end
+end