mss-sdk 1.0.0

data/lib/mss/s3/tree/branch_node.rb

@@ -0,0 +1,67 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+    class Tree
+
+      # Represents a branch in an {S3::Tree}.  From a branch node you
+      # can descend deeper into the tree using {Parent#children} or go
+      # back to the parent node using {#parent}.
+      #
+      # When enumerating nodes in an S3 tree keys grouped by a common
+      # prefix are represented as a branch node.
+      #
+      # Branch nodes are often treated like directories.
+      #
+      # @see Tree
+      # @note Generally you do not need to create branch nodes.
+      class BranchNode < Node
+
+        include Parent
+
+        # @api private
+        def initialize parent, collection, options = {}
+          @parent = parent
+          super(collection,
+                options.merge(:prefix => collection.prefix))
+        end
+
+        # @return [Tree, BranchNode] The parent node in the tree.
+        attr_reader :parent
+
+        # @return [true]
+        def branch?
+          true
+        end
+
+        # @return [false]
+        def leaf?
+          false
+        end
+
+        # Returns a new Tree object that starts at this branch node.
+        # The returned tree will have the same prefix, delimiter and
+        # append mode as the tree the branch belongs to.
+        #
+        # @return [Tree]
+        def as_tree
+          Tree.new(collection,
+                   :prefix => prefix,
+                   :delimiter => delimiter,
+                   :append => append?)
+        end
+
+      end
+    end
+  end
+end

data/lib/mss/s3/tree/child_collection.rb

@@ -0,0 +1,103 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+    class Tree
+
+      class ChildCollection
+
+        include Core::Model
+        include Enumerable
+
+        # @api private
+        def initialize parent, collection, options = {}
+
+          options = {
+            :prefix => nil,
+            :delimiter => '/',
+            :append => true,
+          }.merge(options)
+
+          @parent = parent
+          @collection = collection
+          @prefix = options[:prefix]
+          @delimiter = options[:delimiter]
+          @append = options[:append]
+
+          super
+
+        end
+
+        # @return [Tree, BranchNode] The parent node in the tree.
+        attr_reader :parent
+
+        # @return [ObjectCollection, ObjectVersionCollection,
+        #   MultipartUploadCollection] Returns the collection this
+        #   tree is based on.
+        attr_reader :collection
+
+        # A tree may have a prefix of where in the bucket to be based from.
+        # @return [String,nil]
+        attr_reader :prefix
+
+        # When looking at S3 keys as a tree, the delimiter defines what
+        # string pattern seperates each level of the tree.  The delimiter
+        # defaults to '/' (like in a file system).
+        # @return [String]
+        attr_reader :delimiter
+
+        # @return [Boolean] Returns true if the tree is set to auto-append
+        #   the delimiter to the prefix when the prefix does not end with
+        #   the delimiter.
+        def append?
+          @append
+        end
+
+        # Yields up branches and leaves.
+        #
+        # A branch node represents a common prefix (like a directory)
+        # and a leaf node represents a key (S3 object).
+        #
+        # @yield [tree_node] Yields up a mixture of branches and leafs.
+        # @yieldparam [BranchNode,LeafNode] tree_node A branch or a leaf.
+        # @return [nil]
+        def each &block
+          collection = self.collection
+          if prefix = prefix_with_delim
+            collection = collection.with_prefix(prefix)
+          end
+          collection.each(:delimiter => delimiter) do |member|
+            case
+            when member.respond_to?(:key)
+              yield LeafNode.new(parent, member)
+            when member.respond_to?(:prefix)
+              yield BranchNode.new(parent, member,
+                                   :delimiter => delimiter,
+                                   :append => append?)
+            end
+          end
+          nil
+        end
+
+        protected
+        def prefix_with_delim
+          return prefix unless append?
+          return nil if prefix.nil?
+          prefix =~ /#{delimiter}$/ ? prefix : "#{prefix}#{delimiter}"
+        end
+
+      end
+
+    end
+  end
+end

data/lib/mss/s3/tree/leaf_node.rb

@@ -0,0 +1,93 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+    class Tree
+
+      # Represents a leaf in an {S3::Tree}.
+      #
+      # When enumerating nodes in an S3 tree, keys are yielded
+      # as leaf nodes (they have no children beneath them).
+      #
+      # @see Tree
+      # @note Generally you do not need to create leaf nodes
+      class LeafNode < Node
+
+        # @api private
+        def initialize parent, member
+          @parent = parent
+          @member = member
+          super()
+        end
+
+        # @return [Tree, BranchNode] The parent node in the tree.
+        attr_reader :parent
+
+        # @return [mixed] Returns the object this leaf node represents.
+        # @see #object
+        # @see #version
+        # @see #upload
+        attr_reader :member
+
+        # @return [String] the key this leaf node represents.
+        def key
+          @member.key
+        end
+
+        # @return [false]
+        def branch?
+          false
+        end
+
+        # @return [true]
+        def leaf?
+          true
+        end
+
+        # @return [S3Object] The object this leaf node represents.
+        def object
+          if @member.kind_of?(S3Object)
+            @member
+          else
+            @member.object
+          end
+        end
+
+        # @return [ObjectVersion] Returns the object version this leaf
+        #   node represents.
+        def version
+          if @member.kind_of?(ObjectVersion)
+            @member
+          else
+            raise "This leaf does not represent a version"
+          end
+        end
+
+        # @return [MultipartUpload] Returns the object version this leaf
+        #   node represents.
+        def upload
+          if @member.kind_of?(MultipartUpload)
+            @member
+          else
+            raise "This leaf does not represent an upload"
+          end
+        end
+
+        def inspect
+          "<#{self.class}:#{@member.bucket.name}:#{key}>"
+        end
+
+      end
+    end
+  end
+end

data/lib/mss/s3/tree/node.rb

@@ -0,0 +1,21 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+    class Tree
+      # @api private
+      class Node
+      end
+    end
+  end
+end

data/lib/mss/s3/tree/parent.rb

@@ -0,0 +1,86 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    class Tree
+
+      # Common methods for tree nodes that are parents to other nodes
+      # ({Tree} and {BranchNode}).
+      module Parent
+
+        include Core::Model
+
+        # @api private
+        def initialize collection, options = {}
+
+          options = {
+            :prefix => nil,
+            :delimiter => '/',
+            :append => true,
+          }.merge(options)
+
+          @collection = collection
+          @prefix = options[:prefix]
+          @delimiter = options[:delimiter]
+          @append = options[:append]
+
+          super
+
+        end
+
+        # @return [ObjectCollection, BucketVersionCollection,
+        #   MultipartUploadCollection] The collection whose members
+        #   will be explored using the tree.
+        attr_reader :collection
+
+        # A tree may have a prefix of where in the bucket to be based
+        # from.  A value of `nil` means that the tree will include all
+        # objects in the collection.
+        #
+        # @return [String,nil]
+        attr_reader :prefix
+
+        # When looking at S3 keys as a tree, the delimiter defines what
+        # string pattern seperates each level of the tree.  The delimiter
+        # defaults to '/' (like in a file system).
+        #
+        # @return [String]
+        attr_reader :delimiter
+
+        # @return [Boolean] Returns true if the tree is set to auto-append
+        #   the delimiter to the prefix when the prefix does not end with
+        #   the delimiter.
+        def append?
+          @append
+        end
+
+        # @return [Tree::ChildCollection] A collection representing all
+        #   the child nodes of this node.  These may be either
+        #   {Tree::BranchNode} objects or {Tree::LeafNode} objects.
+        def children
+          Tree::ChildCollection.new(self, collection,
+                                    :delimiter => delimiter,
+                                    :prefix => prefix,
+                                    :append => append?)
+        end
+
+        def inspect
+          "<#{self.class}:#{collection.bucket.name}:#{prefix}>"
+        end
+
+      end
+
+    end
+  end
+end

data/lib/mss/s3/tree.rb

@@ -0,0 +1,115 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    # A utility class that supports exploring an S3 {Bucket} like a
+    # tree.
+    #
+    # Frequently objects stored in S3 have keys that look like a filesystem
+    # directory structure.
+    #
+    # Given you have a bucket with the following keys:
+    #
+    #     README.txt
+    #     videos/wedding.mpg
+    #     videos/family_reunion.mpg
+    #     photos/2010/house.jpg
+    #     photos/2011/fall/leaves.jpg
+    #     photos/2011/summer/vacation.jpg
+    #     photos/2011/summer/family.jpg
+    #
+    # You might like to explore the contents of this bucket as a tree:
+    #
+    #     tree = bucket.as_tree
+    #
+    #     directories = tree.children.select(&:branch?).collect(&:prefix)
+    #     #=> ['photos', 'videos']
+    #
+    #     files = tree.children.select(&:leaf?).collect(&:key)
+    #     #=> ['README.txt']
+    #
+    # If you want to start further down, pass a prefix to {Bucket#as_tree}:
+    #
+    #     tree = bucket.as_tree(:prefix => 'photos/2011')
+    #
+    #     directories = tree.children.select(&:branch?).collect(&:prefix)
+    #     #=> ['photos/2011/fall', 'photos/2011/summer']
+    #
+    #     files = tree.children.select(&:leaf?).collect(&:key)
+    #     #=> []
+    #
+    # All non-leaf nodes ({Tree} and {Tree::BranchNode} instances)
+    # have a {Tree::Parent#children} method that provides access to
+    # the next level of the tree, and all nodes ({Tree},
+    # {Tree::BranchNode}, and {Tree::LeafNode}) have a {#parent}
+    # method that returns the parent node.  In our examples above, the
+    # non-leaf nodes are common prefixes to multiple keys
+    # (directories) and leaf nodes are object keys.
+    #
+    # You can continue crawling the tree using the `children`
+    # collection on each branch node, which will contain the branch
+    # nodes and leaf nodes below it.
+    #
+    # You can construct a Tree object using the `as_tree` method of
+    # any of the following classes:
+    #
+    # * {Bucket} or {ObjectCollection} (for {S3Object} leaf nodes)
+    #
+    # * {BucketVersionCollection} (for {ObjectVersion} leaf nodes)
+    #
+    # * {MultipartUploadCollection} (for {MultipartUpload} leaf nodes)
+    #
+    # The methods to explore the tree are the same for each kind of
+    # leaf node, but {Tree::LeafNode#member} will return a different
+    # type of object depending on which kind of collection the tree is
+    # using.
+    class Tree
+
+      autoload :BranchNode, 'mss/s3/tree/branch_node'
+      autoload :ChildCollection, 'mss/s3/tree/child_collection'
+      autoload :LeafNode, 'mss/s3/tree/leaf_node'
+      autoload :Node, 'mss/s3/tree/node'
+      autoload :Parent, 'mss/s3/tree/parent'
+
+      include Parent
+
+      # @param [ObjectCollection, BucketVersionCollection,
+      #   MultipartUploadCollection] collection The collection whose
+      #   members will be explored using the tree.
+      #
+      # @param [Hash] options Additional options for constructing the
+      #   tree.
+      #
+      # @option options [String] :prefix (nil) Set prefix to choose
+      #   where the top of the tree will be.  A value of `nil` means
+      #   that the tree will include all objects in the collection.
+      #
+      # @option options [String] :delimiter ('/') The string that
+      #   separates each level of the tree.  This is usually a
+      #   directory separator.
+      #
+      # @option options [Boolean] :append (true) If true, the delimiter is
+      #   appended to the prefix when the prefix does not already end
+      #   with the delimiter.
+      def initialize collection, options = {}
+        super
+      end
+
+      # @return The parent node in the tree.  In the case of a Tree,
+      #   the parent is always nil.
+      def parent; nil; end
+
+    end
+  end
+end

data/lib/mss/s3/uploaded_part.rb

@@ -0,0 +1,81 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    # Represents a part of a multipart upload that has been uploaded
+    # to S3.
+    #
+    # @example Get the total size of the uploaded parts
+    #  upload.parts.inject(0) { |sum, part| sum + part.size }
+    class UploadedPart
+
+      include Core::Model
+
+      # @return [MultipartUpload] The upload to which this belongs.
+      attr_reader :upload
+
+      # @return [Integer] The part number.
+      attr_reader :part_number
+
+      # @api private
+      def initialize(upload, part_number, opts = {})
+        @upload = upload
+        @part_number = part_number
+        @etag = opts[:etag]
+        super
+      end
+
+      def ==(other)
+        other.kind_of?(UploadedPart) and
+          other.upload == upload and
+          other.part_number == part_number
+      end
+      alias_method :eql?, :==
+
+      # @return [Integer] The size of the part as it currently
+      #   exists in S3.
+      def size
+        get_attribute(:size)
+      end
+
+      # @return [DateTime] The time at which the part was last
+      #   modified.
+      def last_modified
+        get_attribute(:last_modified)
+      end
+
+      # @return [String] The ETag of the part.
+      def etag
+        @etag ||= get_attribute(:etag)
+        @etag
+      end
+
+      # @api private
+      private
+      def get_attribute(name)
+        (resp = client.list_parts(:bucket_name => upload.object.bucket.name,
+                                  :key => upload.object.key,
+                                  :upload_id => upload.id,
+                                  :part_number_marker => part_number-1,
+                                  :max_parts => 1) and
+         part = resp.parts.first and
+         part.part_number == part_number and
+         part.send(name)) or
+          raise "part #{part_number} of upload #{upload.id} does not exist"
+      end
+
+    end
+
+  end
+end

data/lib/mss/s3/uploaded_part_collection.rb

@@ -0,0 +1,83 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    # Represents the collection of parts that have been uploaded for
+    # a given multipart upload.  You can get an instance of this
+    # class by calling {MultipartUpload#parts}.
+    #
+    # @example Get the total size of the uploaded parts
+    #  upload.parts.inject(0) { |sum, part| sum + part.size }
+    class UploadedPartCollection
+
+      include Enumerable
+      include Core::Model
+      include PaginatedCollection
+
+      # @return [MultipartUpload] The upload to which the parts belong.
+      attr_reader :upload
+
+      # @api private
+      def initialize(upload, opts = {})
+        @upload = upload
+        super
+      end
+
+      # @return [UploadedPart] An object representing the part with
+      #   the given part number.
+      #
+      # @param [Integer] number The part number.
+      def [](number)
+        UploadedPart.new(upload, number)
+      end
+
+      # @api private
+      protected
+      def each_member_in_page(page, &block)
+        page.parts.each do |part_info|
+          part = UploadedPart.new(upload, part_info.part_number, :etag => part_info.etag)
+          yield(part)
+        end
+      end
+
+      # @api private
+      protected
+      def list_options(options)
+        opts = super
+        opts.merge!(:bucket_name => upload.object.bucket.name,
+                    :key => upload.object.key,
+                    :upload_id => upload.id)
+        opts
+      end
+
+      # @api private
+      protected
+      def limit_param; :max_parts; end
+
+      # @api private
+      protected
+      def list_request(options)
+        client.list_parts(options)
+      end
+
+      # @api private
+      protected
+      def pagination_markers
+        [:part_number_marker]
+      end
+
+    end
+
+  end
+end